Author's Notes¶

May 2021 Update: Support for TensorFlow 2. Note that the results in this notebook differ from those in the book because of randomness in the seed we have set here vs. the original seed for TensorFlow and numpy in the book. This is expected. I encourage you to experiment on your own by changing the seed for TensorFlow and numpy in this notebook.

Note for Mac users: You will need to use the "tensorflow_mac" Conda environment for this notebook instead of the "unsupervisedLearning" Conda environment. Please refer to the README on the GitHub repo for more.

From Hands-on Unsupervised Learning using Python - Chapter 8 - Hands-on Autoencoder

Google Colab Setup¶

Only for users on Google Colab

In [ ]:
# Define functions to connect to Google and change directories
def connectDrive():
    from google.colab import drive
    drive.mount('/content/drive', force_remount=True)

def changeDirectory(path):
    import os
    original_path = os.getcwd()
    os.chdir(path)
    new_path = os.getcwd()
    print("Original path: ",original_path)
    print("New path: ",new_path)

# Connect to Google Drive
connectDrive()

# Change path
changeDirectory("/content/drive/My Drive/github/handson-unsupervised-learning/")
Mounted at /content/drive
Original path:  /content
New path:  /content/drive/My Drive/github/handson-unsupervised-learning

Import Libraries¶

In [ ]:
'''Main'''
import numpy as np
import pandas as pd
import os, time, re
import pickle, gzip

'''Data Viz'''
import matplotlib.pyplot as plt
import matplotlib as mpl
import seaborn as sns
color = sns.color_palette()
%matplotlib inline

'''Data Prep and Model Evaluation'''
from sklearn import preprocessing as pp
from sklearn.model_selection import train_test_split 
from sklearn.model_selection import StratifiedKFold 
from sklearn.metrics import log_loss
from sklearn.metrics import precision_recall_curve, average_precision_score
from sklearn.metrics import roc_curve, auc, roc_auc_score

'''Algos'''
import lightgbm as lgb

'''TensorFlow and Keras'''
import tensorflow as tf
from tensorflow import keras
K = keras.backend

from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import Activation, Dense, Dropout
from tensorflow.keras.layers import BatchNormalization, Input, Lambda
from tensorflow.keras import regularizers
from tensorflow.keras.losses import mse, binary_crossentropy

Check library versions & set seed¶

In [ ]:
import sys, sklearn
print(f'sklearn    {sklearn.__version__}')
print(f'tensorflow {tf.__version__}')
print(f'keras      {keras.__version__}')
print(f'numpy      {np.__version__}')
sklearn    0.22.2.post1
tensorflow 2.4.1
keras      2.4.0
numpy      1.19.5
In [ ]:
# To make the output stable across runs
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# Check use of GPU
if tf.test.gpu_device_name(): 
    print('Default GPU Device: {}'.format(tf.test.gpu_device_name()))
else:
   print("Please install GPU version of TF, if GPU is available.")
Please install GPU version of TF, if GPU is available.

Data Preparation¶

Load the data¶

In [ ]:
current_path = os.getcwd()
file = os.path.sep.join(['', 'datasets', 'credit_card_data', 'credit_card.csv'])
data = pd.read_csv(current_path + file)
In [ ]:
dataX = data.copy().drop(['Class','Time'],axis=1)
dataY = data['Class'].copy()

Split into train and test¶

In [ ]:
X_train, X_test, y_train, y_test = train_test_split(
                                    dataX, dataY, test_size=0.33, 
                                    random_state=2018, stratify=dataY)

Scale the data¶

In [ ]:
# Suppress warnings
pd.set_option('mode.chained_assignment', None)

featuresToScale = dataX.columns
sX = pp.StandardScaler(copy=True, with_mean=True, with_std=True)
X_train.loc[:,featuresToScale] = sX.fit_transform(X_train.loc[:,featuresToScale])
X_test.loc[:,featuresToScale] = sX.transform(X_test.loc[:,featuresToScale])
In [ ]:
X_train_AE, X_test_AE = X_train.copy(), X_test.copy()

Define evaluation function and plotting function¶

In [ ]:
def anomalyScores(originalDF, reducedDF):
    loss = np.sum((np.array(originalDF) - 
                   np.array(reducedDF))**2, axis=1)
    loss = pd.Series(data=loss,index=originalDF.index)
    loss = (loss-np.min(loss))/(np.max(loss)-np.min(loss))
    return loss
In [ ]:
def plotResults(trueLabels, anomalyScores, returnPreds = False):
    preds = pd.concat([trueLabels, anomalyScores], axis=1)
    preds.columns = ['trueLabel', 'anomalyScore']
    precision, recall, thresholds = \
        precision_recall_curve(preds['trueLabel'], 
                               preds['anomalyScore'])
    average_precision = average_precision_score( 
                        preds['trueLabel'], preds['anomalyScore'])
    
    plt.step(recall, precision, color='k', alpha=0.7, where='post')
    plt.fill_between(recall, precision, step='post', alpha=0.3, color='k')

    plt.xlabel('Recall')
    plt.ylabel('Precision')
    plt.ylim([0.0, 1.05])
    plt.xlim([0.0, 1.0])
    
    plt.title('Precision-Recall curve: Average Precision = \
        {0:0.2f}'.format(average_precision))

    fpr, tpr, thresholds = roc_curve(preds['trueLabel'], 
                                     preds['anomalyScore'])
    areaUnderROC = auc(fpr, tpr)

    plt.figure()
    plt.plot(fpr, tpr, color='r', lw=2, label='ROC curve')
    plt.plot([0, 1], [0, 1], color='k', lw=2, linestyle='--')
    plt.xlim([0.0, 1.0])
    plt.ylim([0.0, 1.05])
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    plt.title('Receiver operating characteristic: Area under the \
        curve = {0:0.2f}'.format(areaUnderROC))
    plt.legend(loc="lower right")
    plt.show()
    
    if returnPreds==True:
        return preds, average_precision

Model One¶

Two layer complete autoencoder with linear activation¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# Call neural network API
model = Sequential()

# Apply linear activation function to input layer
# Generate hidden layer with 29 nodes, the same as the input layer
model.add(Dense(units=29, activation='linear',input_dim=29))

# Apply linear activation function to hidden layer
# Generate output layer with 29 nodes
model.add(Dense(units=29, activation='linear'))
In [ ]:
# Compile the model
model.compile(optimizer='adam',
              loss='mean_squared_error',
              metrics=['accuracy'])
In [ ]:
# Train the model
num_epochs = 10
batch_size = 32

history = model.fit(x=X_train_AE, y=X_train_AE,
                    epochs=num_epochs,
                    batch_size=batch_size,
                    shuffle=True,
                    validation_data=(X_train_AE, X_train_AE),
                    verbose=1)
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2891 - accuracy: 0.7103 - val_loss: 0.0011 - val_accuracy: 0.9836
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0014 - accuracy: 0.9889 - val_loss: 1.4438e-05 - val_accuracy: 0.9983
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.7539e-04 - accuracy: 0.9937 - val_loss: 2.6702e-04 - val_accuracy: 0.9944
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.2259e-04 - accuracy: 0.9948 - val_loss: 6.7056e-06 - val_accuracy: 0.9993
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.4883e-04 - accuracy: 0.9931 - val_loss: 5.0246e-05 - val_accuracy: 0.9969
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.7656e-04 - accuracy: 0.9964 - val_loss: 0.0012 - val_accuracy: 0.9860
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.4220e-04 - accuracy: 0.9954 - val_loss: 7.7994e-04 - val_accuracy: 0.9898
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0012 - accuracy: 0.9926 - val_loss: 7.1407e-06 - val_accuracy: 0.9988
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9938 - val_loss: 0.0021 - val_accuracy: 0.9864
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.1810e-04 - accuracy: 0.9934 - val_loss: 0.0028 - val_accuracy: 0.9727

Evaluate on Test Set¶

In [ ]:
predictions = model.predict(X_test, verbose=1)
anomalyScoresAE = anomalyScores(X_test, predictions)
preds = plotResults(y_test, anomalyScoresAE, True)
model.reset_states()
2938/2938 [==============================] - 2s 539us/step
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Apply linear activation function to input layer
    # Generate hidden layer with 29 nodes, the same as the input layer
    model.add(Dense(units=29, activation='linear',input_dim=29))

    # Apply linear activation function to hidden layer
    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f'Mean average precision over 10 runs: {np.mean(test_scores)}')
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3308 - accuracy: 0.6682 - val_loss: 0.0019 - val_accuracy: 0.9778
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0014 - accuracy: 0.9886 - val_loss: 1.6041e-06 - val_accuracy: 0.9994
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.9293e-04 - accuracy: 0.9940 - val_loss: 7.7672e-04 - val_accuracy: 0.9915
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.4527e-04 - accuracy: 0.9943 - val_loss: 2.3835e-06 - val_accuracy: 0.9993
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.0941e-04 - accuracy: 0.9948 - val_loss: 1.2261e-04 - val_accuracy: 0.9946
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.8485e-04 - accuracy: 0.9961 - val_loss: 7.8218e-04 - val_accuracy: 0.9874
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.5350e-04 - accuracy: 0.9958 - val_loss: 0.0021 - val_accuracy: 0.9826
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9903 - val_loss: 8.8468e-06 - val_accuracy: 0.9987
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9938 - val_loss: 0.0028 - val_accuracy: 0.9854
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.7426e-04 - accuracy: 0.9937 - val_loss: 0.0018 - val_accuracy: 0.9800
2938/2938 [==============================] - 2s 525us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2790 - accuracy: 0.7099 - val_loss: 0.0017 - val_accuracy: 0.9748
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9891 - val_loss: 2.8647e-04 - val_accuracy: 0.9948
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.8624e-04 - accuracy: 0.9943 - val_loss: 7.2143e-04 - val_accuracy: 0.9897
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 2.8828e-04 - accuracy: 0.9955 - val_loss: 3.5368e-06 - val_accuracy: 0.9994
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 4.7223e-04 - accuracy: 0.9943 - val_loss: 9.0208e-05 - val_accuracy: 0.9959
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.4415e-04 - accuracy: 0.9968 - val_loss: 6.9744e-04 - val_accuracy: 0.9898
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.2752e-04 - accuracy: 0.9959 - val_loss: 0.0022 - val_accuracy: 0.9872
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0026 - accuracy: 0.9909 - val_loss: 5.5084e-05 - val_accuracy: 0.9971
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0012 - accuracy: 0.9939 - val_loss: 0.0019 - val_accuracy: 0.9880
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.5812e-04 - accuracy: 0.9934 - val_loss: 0.0028 - val_accuracy: 0.9727
2938/2938 [==============================] - 2s 550us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2838 - accuracy: 0.7271 - val_loss: 0.0022 - val_accuracy: 0.9832
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0030 - accuracy: 0.9827 - val_loss: 4.5142e-04 - val_accuracy: 0.9929
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.6377e-04 - accuracy: 0.9923 - val_loss: 7.0631e-04 - val_accuracy: 0.9929
Epoch 4/10
5964/5964 [==============================] - 11s 2ms/step - loss: 3.5409e-04 - accuracy: 0.9956 - val_loss: 0.0011 - val_accuracy: 0.9924
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.9101e-04 - accuracy: 0.9920 - val_loss: 3.3091e-05 - val_accuracy: 0.9970
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 2.9661e-04 - accuracy: 0.9965 - val_loss: 5.1622e-04 - val_accuracy: 0.9892
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.4047e-04 - accuracy: 0.9956 - val_loss: 0.0020 - val_accuracy: 0.9859
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0028 - accuracy: 0.9917 - val_loss: 9.0804e-06 - val_accuracy: 0.9987
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.8690e-04 - accuracy: 0.9944 - val_loss: 4.9165e-04 - val_accuracy: 0.9931
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.6118e-04 - accuracy: 0.9941 - val_loss: 0.0035 - val_accuracy: 0.9720
2938/2938 [==============================] - 2s 527us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3062 - accuracy: 0.7099 - val_loss: 0.0011 - val_accuracy: 0.9803
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0012 - accuracy: 0.9918 - val_loss: 7.9635e-05 - val_accuracy: 0.9960
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 7.2936e-04 - accuracy: 0.9940 - val_loss: 8.3035e-04 - val_accuracy: 0.9911
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.8715e-04 - accuracy: 0.9941 - val_loss: 2.3968e-06 - val_accuracy: 0.9994
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.1669e-04 - accuracy: 0.9943 - val_loss: 7.1605e-05 - val_accuracy: 0.9960
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.7103e-04 - accuracy: 0.9968 - val_loss: 0.0012 - val_accuracy: 0.9832
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.7448e-04 - accuracy: 0.9951 - val_loss: 0.0012 - val_accuracy: 0.9829
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0012 - accuracy: 0.9934 - val_loss: 2.0509e-05 - val_accuracy: 0.9980
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9943 - val_loss: 8.7244e-04 - val_accuracy: 0.9902
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.3660e-04 - accuracy: 0.9948 - val_loss: 0.0017 - val_accuracy: 0.9804
2938/2938 [==============================] - 2s 512us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3046 - accuracy: 0.6929 - val_loss: 0.0017 - val_accuracy: 0.9824
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9866 - val_loss: 4.5951e-06 - val_accuracy: 0.9989
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.2834e-04 - accuracy: 0.9943 - val_loss: 4.2257e-04 - val_accuracy: 0.9913
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 3.0219e-04 - accuracy: 0.9953 - val_loss: 2.6139e-06 - val_accuracy: 0.9995
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.7480e-04 - accuracy: 0.9935 - val_loss: 1.1055e-04 - val_accuracy: 0.9953
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.1186e-04 - accuracy: 0.9969 - val_loss: 8.6972e-04 - val_accuracy: 0.9880
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 2.9375e-04 - accuracy: 0.9958 - val_loss: 2.5760e-04 - val_accuracy: 0.9950
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9923 - val_loss: 4.5907e-05 - val_accuracy: 0.9976
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 9.1548e-04 - accuracy: 0.9946 - val_loss: 0.0017 - val_accuracy: 0.9857
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.1128e-04 - accuracy: 0.9931 - val_loss: 0.0022 - val_accuracy: 0.9772
2938/2938 [==============================] - 2s 574us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2864 - accuracy: 0.7209 - val_loss: 0.0023 - val_accuracy: 0.9791
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9823 - val_loss: 5.3693e-04 - val_accuracy: 0.9903
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 9.4016e-04 - accuracy: 0.9920 - val_loss: 7.0501e-04 - val_accuracy: 0.9907
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 3.4446e-04 - accuracy: 0.9951 - val_loss: 1.1085e-06 - val_accuracy: 0.9996
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 7.7064e-04 - accuracy: 0.9937 - val_loss: 4.0008e-05 - val_accuracy: 0.9973
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 2.6982e-04 - accuracy: 0.9968 - val_loss: 0.0014 - val_accuracy: 0.9852
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 3.1062e-04 - accuracy: 0.9954 - val_loss: 0.0021 - val_accuracy: 0.9839
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0020 - accuracy: 0.9914 - val_loss: 1.0208e-05 - val_accuracy: 0.9987
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 9.2415e-04 - accuracy: 0.9937 - val_loss: 0.0018 - val_accuracy: 0.9887
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.8325e-04 - accuracy: 0.9933 - val_loss: 0.0020 - val_accuracy: 0.9777
2938/2938 [==============================] - 2s 529us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2835 - accuracy: 0.7168 - val_loss: 0.0016 - val_accuracy: 0.9818
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0015 - accuracy: 0.9890 - val_loss: 1.2850e-04 - val_accuracy: 0.9956
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9922 - val_loss: 8.5453e-04 - val_accuracy: 0.9898
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.5226e-04 - accuracy: 0.9954 - val_loss: 8.6625e-07 - val_accuracy: 0.9996
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 4.0864e-04 - accuracy: 0.9951 - val_loss: 2.8528e-05 - val_accuracy: 0.9979
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.0082e-04 - accuracy: 0.9969 - val_loss: 9.7033e-04 - val_accuracy: 0.9865
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 2.7527e-04 - accuracy: 0.9955 - val_loss: 0.0013 - val_accuracy: 0.9893
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9917 - val_loss: 1.0045e-05 - val_accuracy: 0.9986
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 9.6767e-04 - accuracy: 0.9933 - val_loss: 0.0015 - val_accuracy: 0.9894
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.1410e-04 - accuracy: 0.9941 - val_loss: 0.0032 - val_accuracy: 0.9726
2938/2938 [==============================] - 2s 535us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2762 - accuracy: 0.7164 - val_loss: 0.0021 - val_accuracy: 0.9776
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9857 - val_loss: 6.1264e-04 - val_accuracy: 0.9897
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0011 - accuracy: 0.9922 - val_loss: 3.0597e-04 - val_accuracy: 0.9933
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.7408e-04 - accuracy: 0.9952 - val_loss: 4.0847e-06 - val_accuracy: 0.9994
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.1425e-04 - accuracy: 0.9938 - val_loss: 2.3426e-04 - val_accuracy: 0.9941
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 2.6960e-04 - accuracy: 0.9970 - val_loss: 0.0012 - val_accuracy: 0.9826
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 4.0239e-04 - accuracy: 0.9952 - val_loss: 9.8072e-04 - val_accuracy: 0.9903
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9910 - val_loss: 4.5014e-06 - val_accuracy: 0.9991
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 9.4009e-04 - accuracy: 0.9937 - val_loss: 6.8393e-04 - val_accuracy: 0.9929
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.6423e-04 - accuracy: 0.9936 - val_loss: 0.0042 - val_accuracy: 0.9703
2938/2938 [==============================] - 2s 536us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2753 - accuracy: 0.7154 - val_loss: 0.0015 - val_accuracy: 0.9747
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0012 - accuracy: 0.9889 - val_loss: 1.7359e-04 - val_accuracy: 0.9939
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.4449e-04 - accuracy: 0.9939 - val_loss: 5.6167e-04 - val_accuracy: 0.9902
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.4783e-04 - accuracy: 0.9931 - val_loss: 4.2933e-06 - val_accuracy: 0.9993
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 4.5900e-04 - accuracy: 0.9935 - val_loss: 4.8684e-05 - val_accuracy: 0.9971
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 2.3541e-04 - accuracy: 0.9966 - val_loss: 0.0013 - val_accuracy: 0.9858
Epoch 7/10
5964/5964 [==============================] - 11s 2ms/step - loss: 2.8602e-04 - accuracy: 0.9956 - val_loss: 0.0023 - val_accuracy: 0.9857
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0018 - accuracy: 0.9929 - val_loss: 8.3821e-06 - val_accuracy: 0.9987
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0012 - accuracy: 0.9932 - val_loss: 0.0056 - val_accuracy: 0.9770
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.4442e-04 - accuracy: 0.9929 - val_loss: 0.0016 - val_accuracy: 0.9810
2938/2938 [==============================] - 2s 554us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2779 - accuracy: 0.7227 - val_loss: 0.0020 - val_accuracy: 0.9784
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9876 - val_loss: 3.9276e-05 - val_accuracy: 0.9974
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 4.3892e-04 - accuracy: 0.9946 - val_loss: 5.2945e-04 - val_accuracy: 0.9918
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.5143e-04 - accuracy: 0.9939 - val_loss: 2.3280e-04 - val_accuracy: 0.9966
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.2027e-04 - accuracy: 0.9925 - val_loss: 3.1208e-04 - val_accuracy: 0.9935
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 2.1241e-04 - accuracy: 0.9969 - val_loss: 0.0015 - val_accuracy: 0.9842
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 4.1192e-04 - accuracy: 0.9950 - val_loss: 0.0015 - val_accuracy: 0.9863
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0017 - accuracy: 0.9925 - val_loss: 4.9357e-05 - val_accuracy: 0.9970
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0022 - accuracy: 0.9938 - val_loss: 0.0023 - val_accuracy: 0.9855
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.8633e-04 - accuracy: 0.9932 - val_loss: 0.0026 - val_accuracy: 0.9746
2938/2938 [==============================] - 2s 506us/step
Mean average precision over 10 runs: 0.02877491279565572
Out[ ]:
[0.0372, 0.0166, 0.069, 0.0213, 0.0234, 0.017, 0.0195, 0.0263, 0.0247, 0.0328]
In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0288
Coefficient of variation over 10 runs: 0.5141
Out[ ]:
[0.0372, 0.0166, 0.069, 0.0213, 0.0234, 0.017, 0.0195, 0.0263, 0.0247, 0.0328]

Model Two¶

Two layer undercomplete autoencoder with linear activation¶

20 nodes in hidden layer¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Apply linear activation function to input layer
    # Generate hidden layer with 20 nodes
    model.add(Dense(units=20, activation='linear',input_dim=29))

    # Apply linear activation function to hidden layer
    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5200 - accuracy: 0.4822 - val_loss: 0.2780 - val_accuracy: 0.6435
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2821 - accuracy: 0.6348 - val_loss: 0.2753 - val_accuracy: 0.6338
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2869 - accuracy: 0.6418 - val_loss: 0.2740 - val_accuracy: 0.6413
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2711 - accuracy: 0.6485 - val_loss: 0.2744 - val_accuracy: 0.6533
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2784 - accuracy: 0.6443 - val_loss: 0.2732 - val_accuracy: 0.6609
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2679 - accuracy: 0.6644 - val_loss: 0.2735 - val_accuracy: 0.6584
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2713 - accuracy: 0.6646 - val_loss: 0.2736 - val_accuracy: 0.6645
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2903 - accuracy: 0.6659 - val_loss: 0.2737 - val_accuracy: 0.6753
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2982 - accuracy: 0.6750 - val_loss: 0.2739 - val_accuracy: 0.6812
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2689 - accuracy: 0.6813 - val_loss: 0.2739 - val_accuracy: 0.6793
2938/2938 [==============================] - 2s 520us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5172 - accuracy: 0.4620 - val_loss: 0.2778 - val_accuracy: 0.6169
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2805 - accuracy: 0.6176 - val_loss: 0.2752 - val_accuracy: 0.6392
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2830 - accuracy: 0.6379 - val_loss: 0.2740 - val_accuracy: 0.6431
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2721 - accuracy: 0.6456 - val_loss: 0.2746 - val_accuracy: 0.6387
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2786 - accuracy: 0.6427 - val_loss: 0.2733 - val_accuracy: 0.6491
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2687 - accuracy: 0.6451 - val_loss: 0.2738 - val_accuracy: 0.6531
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2704 - accuracy: 0.6534 - val_loss: 0.2741 - val_accuracy: 0.6577
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2901 - accuracy: 0.6610 - val_loss: 0.2739 - val_accuracy: 0.6585
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2988 - accuracy: 0.6594 - val_loss: 0.2744 - val_accuracy: 0.6629
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2692 - accuracy: 0.6656 - val_loss: 0.2734 - val_accuracy: 0.6599
2938/2938 [==============================] - 2s 517us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5168 - accuracy: 0.4578 - val_loss: 0.2791 - val_accuracy: 0.6320
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2830 - accuracy: 0.6350 - val_loss: 0.2757 - val_accuracy: 0.6553
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2862 - accuracy: 0.6636 - val_loss: 0.2743 - val_accuracy: 0.6696
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2724 - accuracy: 0.6649 - val_loss: 0.2745 - val_accuracy: 0.6668
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2800 - accuracy: 0.6628 - val_loss: 0.2736 - val_accuracy: 0.6672
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2686 - accuracy: 0.6587 - val_loss: 0.2739 - val_accuracy: 0.6647
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2717 - accuracy: 0.6664 - val_loss: 0.2742 - val_accuracy: 0.6625
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2919 - accuracy: 0.6657 - val_loss: 0.2740 - val_accuracy: 0.6645
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2989 - accuracy: 0.6670 - val_loss: 0.2744 - val_accuracy: 0.6674
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2696 - accuracy: 0.6688 - val_loss: 0.2736 - val_accuracy: 0.6679
2938/2938 [==============================] - 2s 510us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5146 - accuracy: 0.4445 - val_loss: 0.2784 - val_accuracy: 0.6037
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2818 - accuracy: 0.6062 - val_loss: 0.2756 - val_accuracy: 0.6261
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2882 - accuracy: 0.6349 - val_loss: 0.2741 - val_accuracy: 0.6378
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2726 - accuracy: 0.6464 - val_loss: 0.2744 - val_accuracy: 0.6394
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2783 - accuracy: 0.6344 - val_loss: 0.2733 - val_accuracy: 0.6414
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2692 - accuracy: 0.6362 - val_loss: 0.2740 - val_accuracy: 0.6414
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2712 - accuracy: 0.6365 - val_loss: 0.2743 - val_accuracy: 0.6443
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2914 - accuracy: 0.6492 - val_loss: 0.2737 - val_accuracy: 0.6506
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2972 - accuracy: 0.6476 - val_loss: 0.2742 - val_accuracy: 0.6539
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2697 - accuracy: 0.6560 - val_loss: 0.2738 - val_accuracy: 0.6573
2938/2938 [==============================] - 2s 512us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5362 - accuracy: 0.4355 - val_loss: 0.2780 - val_accuracy: 0.5982
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2817 - accuracy: 0.6054 - val_loss: 0.2753 - val_accuracy: 0.6117
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2835 - accuracy: 0.6164 - val_loss: 0.2743 - val_accuracy: 0.6300
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2733 - accuracy: 0.6422 - val_loss: 0.2748 - val_accuracy: 0.6354
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2795 - accuracy: 0.6332 - val_loss: 0.2736 - val_accuracy: 0.6428
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2690 - accuracy: 0.6410 - val_loss: 0.2740 - val_accuracy: 0.6416
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2711 - accuracy: 0.6424 - val_loss: 0.2742 - val_accuracy: 0.6474
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2899 - accuracy: 0.6492 - val_loss: 0.2741 - val_accuracy: 0.6484
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2973 - accuracy: 0.6503 - val_loss: 0.2745 - val_accuracy: 0.6548
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2696 - accuracy: 0.6630 - val_loss: 0.2736 - val_accuracy: 0.6534
2938/2938 [==============================] - 2s 514us/step
Epoch 1/10
5964/5964 [==============================] - 12s 2ms/step - loss: 0.5290 - accuracy: 0.4867 - val_loss: 0.2791 - val_accuracy: 0.6367
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2820 - accuracy: 0.6318 - val_loss: 0.2761 - val_accuracy: 0.6496
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2856 - accuracy: 0.6561 - val_loss: 0.2749 - val_accuracy: 0.6517
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2739 - accuracy: 0.6584 - val_loss: 0.2748 - val_accuracy: 0.6539
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2799 - accuracy: 0.6559 - val_loss: 0.2735 - val_accuracy: 0.6597
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2685 - accuracy: 0.6561 - val_loss: 0.2741 - val_accuracy: 0.6692
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2711 - accuracy: 0.6701 - val_loss: 0.2742 - val_accuracy: 0.6809
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2908 - accuracy: 0.6835 - val_loss: 0.2740 - val_accuracy: 0.6887
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2982 - accuracy: 0.6870 - val_loss: 0.2743 - val_accuracy: 0.6915
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2698 - accuracy: 0.6941 - val_loss: 0.2736 - val_accuracy: 0.6894
2938/2938 [==============================] - 2s 513us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5167 - accuracy: 0.4460 - val_loss: 0.2788 - val_accuracy: 0.5749
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2816 - accuracy: 0.5687 - val_loss: 0.2759 - val_accuracy: 0.5731
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2859 - accuracy: 0.5806 - val_loss: 0.2746 - val_accuracy: 0.5731
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2740 - accuracy: 0.5857 - val_loss: 0.2749 - val_accuracy: 0.5876
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2802 - accuracy: 0.5914 - val_loss: 0.2738 - val_accuracy: 0.6063
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2701 - accuracy: 0.6020 - val_loss: 0.2744 - val_accuracy: 0.6095
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2712 - accuracy: 0.6125 - val_loss: 0.2753 - val_accuracy: 0.6252
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2918 - accuracy: 0.6334 - val_loss: 0.2739 - val_accuracy: 0.6328
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2971 - accuracy: 0.6334 - val_loss: 0.2745 - val_accuracy: 0.6443
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2703 - accuracy: 0.6430 - val_loss: 0.2738 - val_accuracy: 0.6521
2938/2938 [==============================] - 2s 534us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5696 - accuracy: 0.4388 - val_loss: 0.2783 - val_accuracy: 0.6143
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2801 - accuracy: 0.6045 - val_loss: 0.2753 - val_accuracy: 0.6202
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2860 - accuracy: 0.6201 - val_loss: 0.2740 - val_accuracy: 0.6374
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2730 - accuracy: 0.6566 - val_loss: 0.2743 - val_accuracy: 0.6483
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2791 - accuracy: 0.6471 - val_loss: 0.2730 - val_accuracy: 0.6524
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2681 - accuracy: 0.6522 - val_loss: 0.2736 - val_accuracy: 0.6477
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2712 - accuracy: 0.6558 - val_loss: 0.2736 - val_accuracy: 0.6614
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2902 - accuracy: 0.6650 - val_loss: 0.2737 - val_accuracy: 0.6683
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2974 - accuracy: 0.6713 - val_loss: 0.2738 - val_accuracy: 0.6687
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2691 - accuracy: 0.6710 - val_loss: 0.2734 - val_accuracy: 0.6711
2938/2938 [==============================] - 2s 543us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5008 - accuracy: 0.4727 - val_loss: 0.2786 - val_accuracy: 0.6051
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2817 - accuracy: 0.6110 - val_loss: 0.2761 - val_accuracy: 0.6221
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2881 - accuracy: 0.6389 - val_loss: 0.2745 - val_accuracy: 0.6381
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2737 - accuracy: 0.6440 - val_loss: 0.2746 - val_accuracy: 0.6432
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2783 - accuracy: 0.6418 - val_loss: 0.2734 - val_accuracy: 0.6472
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2682 - accuracy: 0.6459 - val_loss: 0.2739 - val_accuracy: 0.6498
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2711 - accuracy: 0.6565 - val_loss: 0.2737 - val_accuracy: 0.6567
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2904 - accuracy: 0.6615 - val_loss: 0.2736 - val_accuracy: 0.6627
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2982 - accuracy: 0.6640 - val_loss: 0.2739 - val_accuracy: 0.6643
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2690 - accuracy: 0.6656 - val_loss: 0.2735 - val_accuracy: 0.6685
2938/2938 [==============================] - 2s 534us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5229 - accuracy: 0.4444 - val_loss: 0.2801 - val_accuracy: 0.5858
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2817 - accuracy: 0.5887 - val_loss: 0.2768 - val_accuracy: 0.6055
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2882 - accuracy: 0.6165 - val_loss: 0.2752 - val_accuracy: 0.6374
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2730 - accuracy: 0.6495 - val_loss: 0.2750 - val_accuracy: 0.6591
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2797 - accuracy: 0.6565 - val_loss: 0.2737 - val_accuracy: 0.6645
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2681 - accuracy: 0.6602 - val_loss: 0.2743 - val_accuracy: 0.6672
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2720 - accuracy: 0.6661 - val_loss: 0.2742 - val_accuracy: 0.6706
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2921 - accuracy: 0.6697 - val_loss: 0.2739 - val_accuracy: 0.6696
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.2989 - accuracy: 0.6734 - val_loss: 0.2748 - val_accuracy: 0.6710
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2693 - accuracy: 0.6708 - val_loss: 0.2736 - val_accuracy: 0.6626
2938/2938 [==============================] - 2s 515us/step
Mean average precision over 10 runs: 0.2752
Out[ ]:
[0.2799, 0.2837, 0.2649, 0.2822, 0.2744, 0.2674, 0.2816, 0.2711, 0.2755, 0.271]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.2752
Coefficient of variation over 10 runs: 0.0225
Out[ ]:
[0.2799, 0.2837, 0.2649, 0.2822, 0.2744, 0.2674, 0.2816, 0.2711, 0.2755, 0.271]

Model Two v2¶

Two layer undercomplete autoencoder with linear activation¶

With 27 nodes this time¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Apply linear activation function to input layer
    # Generate hidden layer with 27 nodes
    model.add(Dense(units=27, activation='linear',input_dim=29))

    # Apply linear activation function to hidden layer
    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3209 - accuracy: 0.6425 - val_loss: 0.0365 - val_accuracy: 0.8872
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0368 - accuracy: 0.8912 - val_loss: 0.0373 - val_accuracy: 0.9247
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0474 - accuracy: 0.9257 - val_loss: 0.0374 - val_accuracy: 0.9512
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0393 - accuracy: 0.9540 - val_loss: 0.0377 - val_accuracy: 0.9516
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0382 - accuracy: 0.9545 - val_loss: 0.0363 - val_accuracy: 0.9615
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0360 - accuracy: 0.9592 - val_loss: 0.0376 - val_accuracy: 0.9510
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0352 - accuracy: 0.9583 - val_loss: 0.0382 - val_accuracy: 0.9532
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0547 - accuracy: 0.9486 - val_loss: 0.0371 - val_accuracy: 0.9637
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0572 - accuracy: 0.9560 - val_loss: 0.0404 - val_accuracy: 0.9560
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0329 - accuracy: 0.9620 - val_loss: 0.0384 - val_accuracy: 0.9529
2938/2938 [==============================] - 2s 512us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3188 - accuracy: 0.6564 - val_loss: 0.0373 - val_accuracy: 0.9131
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0402 - accuracy: 0.9055 - val_loss: 0.0358 - val_accuracy: 0.9328
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0408 - accuracy: 0.9228 - val_loss: 0.0358 - val_accuracy: 0.9536
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0353 - accuracy: 0.9527 - val_loss: 0.0368 - val_accuracy: 0.9379
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0368 - accuracy: 0.9439 - val_loss: 0.0356 - val_accuracy: 0.9412
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0357 - accuracy: 0.9494 - val_loss: 0.0362 - val_accuracy: 0.9445
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0359 - accuracy: 0.9500 - val_loss: 0.0375 - val_accuracy: 0.9495
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0493 - accuracy: 0.9458 - val_loss: 0.0388 - val_accuracy: 0.9507
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0530 - accuracy: 0.9493 - val_loss: 0.0386 - val_accuracy: 0.9557
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0329 - accuracy: 0.9614 - val_loss: 0.0380 - val_accuracy: 0.9582
2938/2938 [==============================] - 2s 522us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3392 - accuracy: 0.6583 - val_loss: 0.0374 - val_accuracy: 0.9221
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0401 - accuracy: 0.9374 - val_loss: 0.0365 - val_accuracy: 0.9505
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0455 - accuracy: 0.9475 - val_loss: 0.0371 - val_accuracy: 0.9579
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0379 - accuracy: 0.9606 - val_loss: 0.0380 - val_accuracy: 0.9532
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0380 - accuracy: 0.9549 - val_loss: 0.0364 - val_accuracy: 0.9642
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0360 - accuracy: 0.9613 - val_loss: 0.0372 - val_accuracy: 0.9576
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0348 - accuracy: 0.9600 - val_loss: 0.0367 - val_accuracy: 0.9604
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0546 - accuracy: 0.9508 - val_loss: 0.0371 - val_accuracy: 0.9638
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0580 - accuracy: 0.9594 - val_loss: 0.0387 - val_accuracy: 0.9602
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0327 - accuracy: 0.9637 - val_loss: 0.0379 - val_accuracy: 0.9580
2938/2938 [==============================] - 2s 536us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3200 - accuracy: 0.6504 - val_loss: 0.0380 - val_accuracy: 0.9415
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0396 - accuracy: 0.9543 - val_loss: 0.0369 - val_accuracy: 0.9636
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0482 - accuracy: 0.9591 - val_loss: 0.0378 - val_accuracy: 0.9659
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0379 - accuracy: 0.9673 - val_loss: 0.0384 - val_accuracy: 0.9575
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0380 - accuracy: 0.9605 - val_loss: 0.0365 - val_accuracy: 0.9669
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0361 - accuracy: 0.9614 - val_loss: 0.0383 - val_accuracy: 0.9500
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0350 - accuracy: 0.9600 - val_loss: 0.0369 - val_accuracy: 0.9590
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0541 - accuracy: 0.9519 - val_loss: 0.0373 - val_accuracy: 0.9633
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0582 - accuracy: 0.9596 - val_loss: 0.0389 - val_accuracy: 0.9595
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0327 - accuracy: 0.9628 - val_loss: 0.0378 - val_accuracy: 0.9550
2938/2938 [==============================] - 2s 511us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3337 - accuracy: 0.6521 - val_loss: 0.0375 - val_accuracy: 0.9155
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0404 - accuracy: 0.9305 - val_loss: 0.0369 - val_accuracy: 0.9474
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0467 - accuracy: 0.9467 - val_loss: 0.0375 - val_accuracy: 0.9534
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.9598 - val_loss: 0.0381 - val_accuracy: 0.9586
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0382 - accuracy: 0.9579 - val_loss: 0.0365 - val_accuracy: 0.9662
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0364 - accuracy: 0.9606 - val_loss: 0.0373 - val_accuracy: 0.9535
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0349 - accuracy: 0.9586 - val_loss: 0.0371 - val_accuracy: 0.9593
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0553 - accuracy: 0.9542 - val_loss: 0.0372 - val_accuracy: 0.9644
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0572 - accuracy: 0.9601 - val_loss: 0.0380 - val_accuracy: 0.9605
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0328 - accuracy: 0.9627 - val_loss: 0.0378 - val_accuracy: 0.9594
2938/2938 [==============================] - 2s 533us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3201 - accuracy: 0.6345 - val_loss: 0.0381 - val_accuracy: 0.8996
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.8891 - val_loss: 0.0362 - val_accuracy: 0.9169
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0436 - accuracy: 0.9157 - val_loss: 0.0363 - val_accuracy: 0.9368
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0404 - accuracy: 0.9385 - val_loss: 0.0373 - val_accuracy: 0.9443
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0385 - accuracy: 0.9469 - val_loss: 0.0361 - val_accuracy: 0.9582
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0366 - accuracy: 0.9551 - val_loss: 0.0372 - val_accuracy: 0.9520
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0356 - accuracy: 0.9593 - val_loss: 0.0372 - val_accuracy: 0.9604
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0533 - accuracy: 0.9568 - val_loss: 0.0371 - val_accuracy: 0.9634
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0558 - accuracy: 0.9589 - val_loss: 0.0379 - val_accuracy: 0.9588
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0327 - accuracy: 0.9624 - val_loss: 0.0383 - val_accuracy: 0.9574
2938/2938 [==============================] - 2s 559us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3314 - accuracy: 0.6595 - val_loss: 0.0382 - val_accuracy: 0.8808
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0401 - accuracy: 0.8860 - val_loss: 0.0361 - val_accuracy: 0.9015
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.9021 - val_loss: 0.0360 - val_accuracy: 0.9038
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0380 - accuracy: 0.9135 - val_loss: 0.0370 - val_accuracy: 0.9152
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0376 - accuracy: 0.9180 - val_loss: 0.0362 - val_accuracy: 0.9280
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0372 - accuracy: 0.9300 - val_loss: 0.0375 - val_accuracy: 0.9215
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0351 - accuracy: 0.9334 - val_loss: 0.0371 - val_accuracy: 0.9464
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.9393 - val_loss: 0.0376 - val_accuracy: 0.9548
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0587 - accuracy: 0.9485 - val_loss: 0.0384 - val_accuracy: 0.9564
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0332 - accuracy: 0.9594 - val_loss: 0.0376 - val_accuracy: 0.9588
2938/2938 [==============================] - 2s 561us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3814 - accuracy: 0.6203 - val_loss: 0.0375 - val_accuracy: 0.9055
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0387 - accuracy: 0.9122 - val_loss: 0.0356 - val_accuracy: 0.9285
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0376 - accuracy: 0.9308 - val_loss: 0.0358 - val_accuracy: 0.9359
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0360 - accuracy: 0.9397 - val_loss: 0.0358 - val_accuracy: 0.9446
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0369 - accuracy: 0.9445 - val_loss: 0.0355 - val_accuracy: 0.9546
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0354 - accuracy: 0.9557 - val_loss: 0.0356 - val_accuracy: 0.9547
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0373 - accuracy: 0.9552 - val_loss: 0.0391 - val_accuracy: 0.9473
Epoch 8/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0397 - accuracy: 0.9573 - val_loss: 0.0359 - val_accuracy: 0.9548
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0367 - accuracy: 0.9596 - val_loss: 0.0369 - val_accuracy: 0.9472
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0356 - accuracy: 0.9472 - val_loss: 0.0371 - val_accuracy: 0.9515
2938/2938 [==============================] - 2s 529us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3032 - accuracy: 0.6632 - val_loss: 0.0369 - val_accuracy: 0.9214
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.9375 - val_loss: 0.0366 - val_accuracy: 0.9577
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0470 - accuracy: 0.9517 - val_loss: 0.0380 - val_accuracy: 0.9659
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0381 - accuracy: 0.9665 - val_loss: 0.0379 - val_accuracy: 0.9620
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0385 - accuracy: 0.9651 - val_loss: 0.0365 - val_accuracy: 0.9695
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0362 - accuracy: 0.9641 - val_loss: 0.0370 - val_accuracy: 0.9597
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0347 - accuracy: 0.9635 - val_loss: 0.0379 - val_accuracy: 0.9591
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0549 - accuracy: 0.9568 - val_loss: 0.0372 - val_accuracy: 0.9646
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0578 - accuracy: 0.9583 - val_loss: 0.0378 - val_accuracy: 0.9604
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0328 - accuracy: 0.9633 - val_loss: 0.0378 - val_accuracy: 0.9617
2938/2938 [==============================] - 2s 527us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3304 - accuracy: 0.6521 - val_loss: 0.0374 - val_accuracy: 0.9069
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0394 - accuracy: 0.9077 - val_loss: 0.0358 - val_accuracy: 0.9077
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0367 - accuracy: 0.9132 - val_loss: 0.0352 - val_accuracy: 0.9173
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0336 - accuracy: 0.9296 - val_loss: 0.0355 - val_accuracy: 0.9140
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0379 - accuracy: 0.9180 - val_loss: 0.0343 - val_accuracy: 0.9457
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0312 - accuracy: 0.9466 - val_loss: 0.0354 - val_accuracy: 0.9138
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0342 - accuracy: 0.9378 - val_loss: 0.0373 - val_accuracy: 0.9250
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0354 - accuracy: 0.9372 - val_loss: 0.0359 - val_accuracy: 0.9471
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0373 - accuracy: 0.9517 - val_loss: 0.0355 - val_accuracy: 0.9626
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0361 - accuracy: 0.9599 - val_loss: 0.0353 - val_accuracy: 0.9402
2938/2938 [==============================] - 2s 532us/step
Mean average precision over 10 runs: 0.0821
Out[ ]:
[0.0129, 0.02, 0.0131, 0.0191, 0.0129, 0.0149, 0.0138, 0.0135, 0.0164, 0.6844]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0821
Coefficient of variation over 10 runs: 2.4455
Out[ ]:
[0.0129, 0.02, 0.0131, 0.0191, 0.0129, 0.0149, 0.0138, 0.0135, 0.0164, 0.6844]

Model Three¶

Three layer undercomplete autoencoder with linear activation.¶

With 28 and 27 nodes in the two hidden layers, respectively¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
tf.random.set_seed(42)
np.random.seed(42)
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Apply linear activation function to input layer
    # Generate first hidden layer with 27 nodes
    # Generate second hidden layer with 28 nodes
    model.add(Dense(units=28, activation='linear',input_dim=29))
    model.add(Dense(units=27, activation='linear'))

    # Apply linear activation function to second hidden layer
    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.3168 - accuracy: 0.6229 - val_loss: 0.0369 - val_accuracy: 0.9341
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.9412 - val_loss: 0.0366 - val_accuracy: 0.9531
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0470 - accuracy: 0.9534 - val_loss: 0.0393 - val_accuracy: 0.9611
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.9632 - val_loss: 0.0384 - val_accuracy: 0.9571
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9581 - val_loss: 0.0363 - val_accuracy: 0.9685
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0370 - accuracy: 0.9622 - val_loss: 0.0390 - val_accuracy: 0.9503
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0355 - accuracy: 0.9588 - val_loss: 0.0373 - val_accuracy: 0.9602
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0553 - accuracy: 0.9596 - val_loss: 0.0377 - val_accuracy: 0.9630
Epoch 9/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0577 - accuracy: 0.9580 - val_loss: 0.0391 - val_accuracy: 0.9571
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0333 - accuracy: 0.9597 - val_loss: 0.0384 - val_accuracy: 0.9564
2938/2938 [==============================] - 2s 523us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2993 - accuracy: 0.6666 - val_loss: 0.0388 - val_accuracy: 0.9175
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.9367 - val_loss: 0.0366 - val_accuracy: 0.9533
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0468 - accuracy: 0.9520 - val_loss: 0.0395 - val_accuracy: 0.9561
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0387 - accuracy: 0.9621 - val_loss: 0.0386 - val_accuracy: 0.9533
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.9561 - val_loss: 0.0364 - val_accuracy: 0.9656
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0364 - accuracy: 0.9621 - val_loss: 0.0375 - val_accuracy: 0.9534
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0352 - accuracy: 0.9600 - val_loss: 0.0391 - val_accuracy: 0.9532
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0561 - accuracy: 0.9560 - val_loss: 0.0375 - val_accuracy: 0.9627
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0586 - accuracy: 0.9573 - val_loss: 0.0386 - val_accuracy: 0.9609
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0335 - accuracy: 0.9615 - val_loss: 0.0389 - val_accuracy: 0.9510
2938/2938 [==============================] - 2s 560us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2930 - accuracy: 0.6313 - val_loss: 0.0420 - val_accuracy: 0.8860
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.8955 - val_loss: 0.0365 - val_accuracy: 0.9158
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0417 - accuracy: 0.9190 - val_loss: 0.0406 - val_accuracy: 0.9272
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0382 - accuracy: 0.9404 - val_loss: 0.0374 - val_accuracy: 0.9426
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0386 - accuracy: 0.9447 - val_loss: 0.0358 - val_accuracy: 0.9541
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0367 - accuracy: 0.9516 - val_loss: 0.0399 - val_accuracy: 0.9402
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0360 - accuracy: 0.9530 - val_loss: 0.0386 - val_accuracy: 0.9562
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0532 - accuracy: 0.9575 - val_loss: 0.0374 - val_accuracy: 0.9643
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0585 - accuracy: 0.9591 - val_loss: 0.0384 - val_accuracy: 0.9586
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0334 - accuracy: 0.9611 - val_loss: 0.0388 - val_accuracy: 0.9568
2938/2938 [==============================] - 2s 529us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3136 - accuracy: 0.6349 - val_loss: 0.0419 - val_accuracy: 0.9011
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0406 - accuracy: 0.9094 - val_loss: 0.0360 - val_accuracy: 0.9130
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.9085 - val_loss: 0.0373 - val_accuracy: 0.9132
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0386 - accuracy: 0.9240 - val_loss: 0.0377 - val_accuracy: 0.9314
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0390 - accuracy: 0.9342 - val_loss: 0.0356 - val_accuracy: 0.9473
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0367 - accuracy: 0.9470 - val_loss: 0.0370 - val_accuracy: 0.9432
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0363 - accuracy: 0.9501 - val_loss: 0.0378 - val_accuracy: 0.9524
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.9471 - val_loss: 0.0376 - val_accuracy: 0.9602
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0539 - accuracy: 0.9586 - val_loss: 0.0380 - val_accuracy: 0.9619
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0334 - accuracy: 0.9626 - val_loss: 0.0428 - val_accuracy: 0.9463
2938/2938 [==============================] - 2s 539us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2715 - accuracy: 0.6668 - val_loss: 0.0383 - val_accuracy: 0.9031
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.9190 - val_loss: 0.0365 - val_accuracy: 0.9249
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0439 - accuracy: 0.9223 - val_loss: 0.0394 - val_accuracy: 0.9306
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0393 - accuracy: 0.9384 - val_loss: 0.0376 - val_accuracy: 0.9408
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0383 - accuracy: 0.9393 - val_loss: 0.0365 - val_accuracy: 0.9477
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0369 - accuracy: 0.9454 - val_loss: 0.0380 - val_accuracy: 0.9413
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0347 - accuracy: 0.9481 - val_loss: 0.0385 - val_accuracy: 0.9539
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0547 - accuracy: 0.9499 - val_loss: 0.0380 - val_accuracy: 0.9618
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0596 - accuracy: 0.9588 - val_loss: 0.0391 - val_accuracy: 0.9599
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0331 - accuracy: 0.9621 - val_loss: 0.0386 - val_accuracy: 0.9554
2938/2938 [==============================] - 2s 560us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3235 - accuracy: 0.6349 - val_loss: 0.0383 - val_accuracy: 0.9317
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0392 - accuracy: 0.9308 - val_loss: 0.0358 - val_accuracy: 0.9389
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.9375 - val_loss: 0.0384 - val_accuracy: 0.9506
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.9519 - val_loss: 0.0369 - val_accuracy: 0.9583
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9575 - val_loss: 0.0359 - val_accuracy: 0.9681
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0370 - accuracy: 0.9634 - val_loss: 0.0383 - val_accuracy: 0.9514
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0365 - accuracy: 0.9616 - val_loss: 0.0380 - val_accuracy: 0.9581
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0521 - accuracy: 0.9608 - val_loss: 0.0373 - val_accuracy: 0.9654
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0521 - accuracy: 0.9653 - val_loss: 0.0386 - val_accuracy: 0.9611
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0335 - accuracy: 0.9618 - val_loss: 0.0395 - val_accuracy: 0.9499
2938/2938 [==============================] - 2s 527us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2988 - accuracy: 0.6713 - val_loss: 0.0373 - val_accuracy: 0.9022
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0401 - accuracy: 0.9081 - val_loss: 0.0346 - val_accuracy: 0.9387
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0351 - accuracy: 0.9431 - val_loss: 0.0344 - val_accuracy: 0.9279
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0318 - accuracy: 0.9483 - val_loss: 0.0363 - val_accuracy: 0.9274
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.9297 - val_loss: 0.0348 - val_accuracy: 0.9482
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0325 - accuracy: 0.9494 - val_loss: 0.0384 - val_accuracy: 0.9256
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0348 - accuracy: 0.9461 - val_loss: 0.0366 - val_accuracy: 0.9236
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0373 - accuracy: 0.9260 - val_loss: 0.0365 - val_accuracy: 0.9445
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0410 - accuracy: 0.9481 - val_loss: 0.0357 - val_accuracy: 0.9665
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0367 - accuracy: 0.9615 - val_loss: 0.0380 - val_accuracy: 0.9391
2938/2938 [==============================] - 2s 619us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.3382 - accuracy: 0.6437 - val_loss: 0.0412 - val_accuracy: 0.9233
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9396 - val_loss: 0.0359 - val_accuracy: 0.9503
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.9504 - val_loss: 0.0369 - val_accuracy: 0.9564
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0385 - accuracy: 0.9573 - val_loss: 0.0373 - val_accuracy: 0.9547
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0395 - accuracy: 0.9568 - val_loss: 0.0360 - val_accuracy: 0.9696
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0373 - accuracy: 0.9630 - val_loss: 0.0369 - val_accuracy: 0.9613
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0361 - accuracy: 0.9620 - val_loss: 0.0398 - val_accuracy: 0.9581
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0544 - accuracy: 0.9579 - val_loss: 0.0372 - val_accuracy: 0.9651
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0564 - accuracy: 0.9607 - val_loss: 0.0381 - val_accuracy: 0.9614
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0334 - accuracy: 0.9619 - val_loss: 0.0385 - val_accuracy: 0.9565
2938/2938 [==============================] - 2s 520us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3120 - accuracy: 0.6217 - val_loss: 0.0394 - val_accuracy: 0.9011
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0409 - accuracy: 0.9200 - val_loss: 0.0371 - val_accuracy: 0.9392
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0470 - accuracy: 0.9359 - val_loss: 0.0379 - val_accuracy: 0.9548
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.9559 - val_loss: 0.0376 - val_accuracy: 0.9558
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0386 - accuracy: 0.9535 - val_loss: 0.0362 - val_accuracy: 0.9637
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0370 - accuracy: 0.9580 - val_loss: 0.0399 - val_accuracy: 0.9411
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0361 - accuracy: 0.9567 - val_loss: 0.0399 - val_accuracy: 0.9519
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0559 - accuracy: 0.9515 - val_loss: 0.0373 - val_accuracy: 0.9644
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0575 - accuracy: 0.9603 - val_loss: 0.0390 - val_accuracy: 0.9566
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0335 - accuracy: 0.9600 - val_loss: 0.0426 - val_accuracy: 0.9401
2938/2938 [==============================] - 2s 519us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2936 - accuracy: 0.6258 - val_loss: 0.0375 - val_accuracy: 0.9018
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.9130 - val_loss: 0.0354 - val_accuracy: 0.9232
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0374 - accuracy: 0.9222 - val_loss: 0.0366 - val_accuracy: 0.9320
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0387 - accuracy: 0.9328 - val_loss: 0.0368 - val_accuracy: 0.9403
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0392 - accuracy: 0.9385 - val_loss: 0.0356 - val_accuracy: 0.9538
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0363 - accuracy: 0.9489 - val_loss: 0.0393 - val_accuracy: 0.9390
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0368 - accuracy: 0.9514 - val_loss: 0.0398 - val_accuracy: 0.9532
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0518 - accuracy: 0.9553 - val_loss: 0.0374 - val_accuracy: 0.9608
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0542 - accuracy: 0.9621 - val_loss: 0.0379 - val_accuracy: 0.9599
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0332 - accuracy: 0.9614 - val_loss: 0.0393 - val_accuracy: 0.9512
2938/2938 [==============================] - 2s 530us/step
Mean average precision over 10 runs: 0.0872
Out[ ]:
[0.0153, 0.0167, 0.0145, 0.03, 0.0255, 0.0154, 0.6637, 0.0234, 0.045, 0.023]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0872
Coefficient of variation over 10 runs: 2.2049
Out[ ]:
[0.0153, 0.0167, 0.0145, 0.03, 0.0255, 0.0154, 0.6637, 0.0234, 0.045, 0.023]

Model Four¶

Four layer undercomplete autoencoder with ReLu activation¶

29 -> 27 -> 22 -> 27 -> 29¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Apply ReLu throughout
    # Generate first hidden layer with 27 nodes
    # Generate second hidden layer with 22 nodes
    model.add(Dense(units=27, activation='relu',input_dim=29))
    model.add(Dense(units=22, activation='relu'))

    # Apply ReLu throughout
    # Generate third hidden layer with 27 nodes
    # Generate output layer with 29 nodes
    model.add(Dense(units=27, activation='relu'))
    model.add(Dense(units=29, activation='relu'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.7648 - accuracy: 0.4436 - val_loss: 0.6323 - val_accuracy: 0.7000
Epoch 2/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.6283 - accuracy: 0.7122 - val_loss: 0.6233 - val_accuracy: 0.7208
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6270 - accuracy: 0.7276 - val_loss: 0.6217 - val_accuracy: 0.7225
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6165 - accuracy: 0.7355 - val_loss: 0.6220 - val_accuracy: 0.7257
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6273 - accuracy: 0.7426 - val_loss: 0.6184 - val_accuracy: 0.7403
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6122 - accuracy: 0.7490 - val_loss: 0.6184 - val_accuracy: 0.7430
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6160 - accuracy: 0.7485 - val_loss: 0.6208 - val_accuracy: 0.7273
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6416 - accuracy: 0.7164 - val_loss: 0.6181 - val_accuracy: 0.7472
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6336 - accuracy: 0.7441 - val_loss: 0.6161 - val_accuracy: 0.7523
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6147 - accuracy: 0.7552 - val_loss: 0.6186 - val_accuracy: 0.7385
2938/2938 [==============================] - 2s 525us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.7414 - accuracy: 0.4732 - val_loss: 0.5855 - val_accuracy: 0.7640
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5778 - accuracy: 0.7840 - val_loss: 0.5662 - val_accuracy: 0.8200
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5689 - accuracy: 0.8230 - val_loss: 0.5668 - val_accuracy: 0.8087
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5543 - accuracy: 0.8290 - val_loss: 0.5589 - val_accuracy: 0.8284
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5619 - accuracy: 0.8342 - val_loss: 0.5572 - val_accuracy: 0.8461
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5502 - accuracy: 0.8435 - val_loss: 0.5587 - val_accuracy: 0.8438
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5529 - accuracy: 0.8431 - val_loss: 0.5575 - val_accuracy: 0.8127
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5648 - accuracy: 0.8447 - val_loss: 0.5540 - val_accuracy: 0.8401
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5722 - accuracy: 0.8460 - val_loss: 0.5529 - val_accuracy: 0.8516
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5491 - accuracy: 0.8579 - val_loss: 0.5558 - val_accuracy: 0.8377
2938/2938 [==============================] - 2s 539us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.7605 - accuracy: 0.4620 - val_loss: 0.6042 - val_accuracy: 0.7434
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5911 - accuracy: 0.7784 - val_loss: 0.5552 - val_accuracy: 0.8269
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5597 - accuracy: 0.8276 - val_loss: 0.5488 - val_accuracy: 0.8244
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5422 - accuracy: 0.8437 - val_loss: 0.5441 - val_accuracy: 0.8536
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5515 - accuracy: 0.8542 - val_loss: 0.5432 - val_accuracy: 0.8506
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5378 - accuracy: 0.8579 - val_loss: 0.5484 - val_accuracy: 0.8544
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5453 - accuracy: 0.8528 - val_loss: 0.5579 - val_accuracy: 0.7720
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5565 - accuracy: 0.8577 - val_loss: 0.5409 - val_accuracy: 0.8617
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5593 - accuracy: 0.8613 - val_loss: 0.5415 - val_accuracy: 0.8543
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5369 - accuracy: 0.8713 - val_loss: 0.5539 - val_accuracy: 0.8551
2938/2938 [==============================] - 2s 528us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.7464 - accuracy: 0.5046 - val_loss: 0.5917 - val_accuracy: 0.8105
Epoch 2/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.5858 - accuracy: 0.8345 - val_loss: 0.5750 - val_accuracy: 0.8530
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5848 - accuracy: 0.8525 - val_loss: 0.5613 - val_accuracy: 0.8445
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5506 - accuracy: 0.8778 - val_loss: 0.5550 - val_accuracy: 0.8728
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5554 - accuracy: 0.8851 - val_loss: 0.5535 - val_accuracy: 0.8840
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5441 - accuracy: 0.8976 - val_loss: 0.5571 - val_accuracy: 0.8486
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5452 - accuracy: 0.9023 - val_loss: 0.5499 - val_accuracy: 0.8807
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5627 - accuracy: 0.8978 - val_loss: 0.5471 - val_accuracy: 0.9020
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5803 - accuracy: 0.8885 - val_loss: 0.5443 - val_accuracy: 0.9204
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5407 - accuracy: 0.9108 - val_loss: 0.5633 - val_accuracy: 0.8704
2938/2938 [==============================] - 2s 539us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.7518 - accuracy: 0.4986 - val_loss: 0.5950 - val_accuracy: 0.7868
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5900 - accuracy: 0.8051 - val_loss: 0.5843 - val_accuracy: 0.8154
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5828 - accuracy: 0.8278 - val_loss: 0.5761 - val_accuracy: 0.8248
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5680 - accuracy: 0.8343 - val_loss: 0.5720 - val_accuracy: 0.8532
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5742 - accuracy: 0.8476 - val_loss: 0.5702 - val_accuracy: 0.8501
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5634 - accuracy: 0.8577 - val_loss: 0.5713 - val_accuracy: 0.8436
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5655 - accuracy: 0.8608 - val_loss: 0.5717 - val_accuracy: 0.8214
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5778 - accuracy: 0.8533 - val_loss: 0.5591 - val_accuracy: 0.8732
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5795 - accuracy: 0.8585 - val_loss: 0.5658 - val_accuracy: 0.8512
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5602 - accuracy: 0.8594 - val_loss: 0.6071 - val_accuracy: 0.7387
2938/2938 [==============================] - 2s 546us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.7448 - accuracy: 0.4881 - val_loss: 0.5892 - val_accuracy: 0.7705
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5845 - accuracy: 0.7938 - val_loss: 0.5703 - val_accuracy: 0.8245
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5757 - accuracy: 0.8273 - val_loss: 0.5682 - val_accuracy: 0.8396
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5590 - accuracy: 0.8400 - val_loss: 0.5771 - val_accuracy: 0.8099
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5684 - accuracy: 0.8430 - val_loss: 0.5636 - val_accuracy: 0.8493
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5569 - accuracy: 0.8556 - val_loss: 0.5677 - val_accuracy: 0.8376
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5589 - accuracy: 0.8624 - val_loss: 0.5625 - val_accuracy: 0.8418
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5694 - accuracy: 0.8720 - val_loss: 0.5579 - val_accuracy: 0.8520
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5824 - accuracy: 0.8569 - val_loss: 0.5586 - val_accuracy: 0.8722
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5542 - accuracy: 0.8844 - val_loss: 0.6013 - val_accuracy: 0.7820
2938/2938 [==============================] - 2s 549us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.7871 - accuracy: 0.4651 - val_loss: 0.6427 - val_accuracy: 0.7778
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6398 - accuracy: 0.7991 - val_loss: 0.6320 - val_accuracy: 0.8188
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.6415 - accuracy: 0.8267 - val_loss: 0.6286 - val_accuracy: 0.8243
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6200 - accuracy: 0.8414 - val_loss: 0.6249 - val_accuracy: 0.8452
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6398 - accuracy: 0.8240 - val_loss: 0.6345 - val_accuracy: 0.8304
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6273 - accuracy: 0.8286 - val_loss: 0.6342 - val_accuracy: 0.8258
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6310 - accuracy: 0.8325 - val_loss: 0.6384 - val_accuracy: 0.7925
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6583 - accuracy: 0.8197 - val_loss: 0.6237 - val_accuracy: 0.8490
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6660 - accuracy: 0.8094 - val_loss: 0.6233 - val_accuracy: 0.8555
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6223 - accuracy: 0.8538 - val_loss: 0.6497 - val_accuracy: 0.7521
2938/2938 [==============================] - 2s 553us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.7682 - accuracy: 0.5059 - val_loss: 0.6147 - val_accuracy: 0.8106
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6062 - accuracy: 0.8525 - val_loss: 0.5867 - val_accuracy: 0.8936
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5918 - accuracy: 0.8832 - val_loss: 0.5907 - val_accuracy: 0.8569
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5802 - accuracy: 0.8891 - val_loss: 0.5830 - val_accuracy: 0.8897
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5947 - accuracy: 0.8844 - val_loss: 0.5821 - val_accuracy: 0.8957
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5792 - accuracy: 0.8938 - val_loss: 0.5812 - val_accuracy: 0.8930
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5823 - accuracy: 0.8926 - val_loss: 0.5928 - val_accuracy: 0.8642
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6041 - accuracy: 0.8821 - val_loss: 0.5823 - val_accuracy: 0.8988
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5974 - accuracy: 0.8877 - val_loss: 0.5801 - val_accuracy: 0.9005
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5841 - accuracy: 0.8872 - val_loss: 0.5875 - val_accuracy: 0.8484
2938/2938 [==============================] - 2s 533us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.7609 - accuracy: 0.4881 - val_loss: 0.6121 - val_accuracy: 0.7783
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6063 - accuracy: 0.8021 - val_loss: 0.6108 - val_accuracy: 0.8282
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6082 - accuracy: 0.8265 - val_loss: 0.5950 - val_accuracy: 0.8263
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5905 - accuracy: 0.8361 - val_loss: 0.5867 - val_accuracy: 0.8493
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5907 - accuracy: 0.8441 - val_loss: 0.5861 - val_accuracy: 0.8408
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5812 - accuracy: 0.8491 - val_loss: 0.5841 - val_accuracy: 0.8580
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5816 - accuracy: 0.8563 - val_loss: 0.5862 - val_accuracy: 0.8304
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5999 - accuracy: 0.8502 - val_loss: 0.5833 - val_accuracy: 0.8543
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6102 - accuracy: 0.8419 - val_loss: 0.5798 - val_accuracy: 0.8623
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5758 - accuracy: 0.8590 - val_loss: 0.5916 - val_accuracy: 0.8038
2938/2938 [==============================] - 2s 520us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.7575 - accuracy: 0.4806 - val_loss: 0.6120 - val_accuracy: 0.7313
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.6058 - accuracy: 0.7513 - val_loss: 0.5911 - val_accuracy: 0.7741
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.5987 - accuracy: 0.7900 - val_loss: 0.5878 - val_accuracy: 0.7875
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5744 - accuracy: 0.8095 - val_loss: 0.5760 - val_accuracy: 0.8019
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5740 - accuracy: 0.8164 - val_loss: 0.5571 - val_accuracy: 0.8411
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5525 - accuracy: 0.8357 - val_loss: 0.5545 - val_accuracy: 0.8443
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5539 - accuracy: 0.8395 - val_loss: 0.5574 - val_accuracy: 0.8051
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5674 - accuracy: 0.8316 - val_loss: 0.5731 - val_accuracy: 0.8301
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5761 - accuracy: 0.8356 - val_loss: 0.5507 - val_accuracy: 0.8587
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.5479 - accuracy: 0.8506 - val_loss: 0.5563 - val_accuracy: 0.8265
2938/2938 [==============================] - 2s 541us/step
Mean average precision over 10 runs: 0.2306
Out[ ]:
[0.2293, 0.2492, 0.2382, 0.2484, 0.2316, 0.2273, 0.1948, 0.2115, 0.223, 0.2526]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.2306
Coefficient of variation over 10 runs: 0.0741
Out[ ]:
[0.2293, 0.2492, 0.2382, 0.2484, 0.2316, 0.2273, 0.1948, 0.2115, 0.223, 0.2526]

Model Five¶

Two layer overcomplete autoencoder with linear activation¶

29 -> 40 -> 29¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Apply linear activation function throughout
    # Generate first hidden layer with 40 nodes
    model.add(Dense(units=40, activation='linear',input_dim=29))

    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()
    
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2050 - accuracy: 0.7910 - val_loss: 0.0011 - val_accuracy: 0.9762
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9896 - val_loss: 3.1096e-04 - val_accuracy: 0.9940
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0014 - accuracy: 0.9922 - val_loss: 0.0011 - val_accuracy: 0.9887
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.9722e-04 - accuracy: 0.9940 - val_loss: 1.0601e-06 - val_accuracy: 0.9995
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.7942e-04 - accuracy: 0.9939 - val_loss: 1.5084e-04 - val_accuracy: 0.9946
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.5940e-04 - accuracy: 0.9963 - val_loss: 0.0018 - val_accuracy: 0.9817
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.4761e-04 - accuracy: 0.9940 - val_loss: 0.0036 - val_accuracy: 0.9795
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0037 - accuracy: 0.9882 - val_loss: 4.8492e-05 - val_accuracy: 0.9963
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0029 - accuracy: 0.9910 - val_loss: 0.0046 - val_accuracy: 0.9776
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.1011e-04 - accuracy: 0.9918 - val_loss: 0.0047 - val_accuracy: 0.9685
2938/2938 [==============================] - 2s 574us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2137 - accuracy: 0.7923 - val_loss: 0.0020 - val_accuracy: 0.9731
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0018 - accuracy: 0.9898 - val_loss: 2.1718e-04 - val_accuracy: 0.9936
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.9673e-04 - accuracy: 0.9927 - val_loss: 8.3525e-04 - val_accuracy: 0.9926
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.8769e-04 - accuracy: 0.9928 - val_loss: 2.6975e-06 - val_accuracy: 0.9993
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.5500e-04 - accuracy: 0.9933 - val_loss: 1.2976e-04 - val_accuracy: 0.9960
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 3.6985e-04 - accuracy: 0.9970 - val_loss: 0.0020 - val_accuracy: 0.9813
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.1426e-04 - accuracy: 0.9937 - val_loss: 0.0019 - val_accuracy: 0.9847
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9904 - val_loss: 1.5503e-05 - val_accuracy: 0.9981
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0018 - accuracy: 0.9926 - val_loss: 0.0059 - val_accuracy: 0.9780
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 9.2538e-04 - accuracy: 0.9918 - val_loss: 0.0035 - val_accuracy: 0.9702
2938/2938 [==============================] - 2s 509us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2071 - accuracy: 0.7888 - val_loss: 0.0024 - val_accuracy: 0.9706
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9902 - val_loss: 3.2068e-04 - val_accuracy: 0.9921
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.7611e-04 - accuracy: 0.9933 - val_loss: 7.4785e-04 - val_accuracy: 0.9914
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.2452e-04 - accuracy: 0.9942 - val_loss: 1.2649e-04 - val_accuracy: 0.9967
Epoch 5/10
5964/5964 [==============================] - 11s 2ms/step - loss: 7.5666e-04 - accuracy: 0.9924 - val_loss: 1.0727e-04 - val_accuracy: 0.9958
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.9379e-04 - accuracy: 0.9971 - val_loss: 0.0013 - val_accuracy: 0.9829
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.4980e-04 - accuracy: 0.9948 - val_loss: 0.0021 - val_accuracy: 0.9855
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9906 - val_loss: 2.3885e-05 - val_accuracy: 0.9976
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0017 - accuracy: 0.9921 - val_loss: 0.0018 - val_accuracy: 0.9888
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.7834e-04 - accuracy: 0.9928 - val_loss: 0.0044 - val_accuracy: 0.9710
2938/2938 [==============================] - 2s 500us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2383 - accuracy: 0.7792 - val_loss: 0.0107 - val_accuracy: 0.9478
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9897 - val_loss: 1.5511e-05 - val_accuracy: 0.9983
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.0681e-04 - accuracy: 0.9938 - val_loss: 8.4162e-04 - val_accuracy: 0.9918
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.6346e-04 - accuracy: 0.9944 - val_loss: 2.0078e-06 - val_accuracy: 0.9995
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.8650e-04 - accuracy: 0.9936 - val_loss: 1.3419e-04 - val_accuracy: 0.9960
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.2295e-04 - accuracy: 0.9971 - val_loss: 0.0011 - val_accuracy: 0.9862
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.4715e-04 - accuracy: 0.9953 - val_loss: 0.0020 - val_accuracy: 0.9865
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0022 - accuracy: 0.9923 - val_loss: 1.9046e-05 - val_accuracy: 0.9985
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0017 - accuracy: 0.9937 - val_loss: 0.0014 - val_accuracy: 0.9905
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 9.4095e-04 - accuracy: 0.9930 - val_loss: 0.0011 - val_accuracy: 0.9819
2938/2938 [==============================] - 2s 500us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2179 - accuracy: 0.7812 - val_loss: 0.0039 - val_accuracy: 0.9601
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0015 - accuracy: 0.9895 - val_loss: 2.6589e-04 - val_accuracy: 0.9935
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9928 - val_loss: 0.0012 - val_accuracy: 0.9878
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.5432e-04 - accuracy: 0.9945 - val_loss: 5.0071e-06 - val_accuracy: 0.9991
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.0897e-04 - accuracy: 0.9938 - val_loss: 1.2628e-04 - val_accuracy: 0.9948
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.7272e-04 - accuracy: 0.9961 - val_loss: 5.3590e-04 - val_accuracy: 0.9887
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.4928e-04 - accuracy: 0.9943 - val_loss: 0.0015 - val_accuracy: 0.9860
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9912 - val_loss: 2.7145e-05 - val_accuracy: 0.9974
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9928 - val_loss: 0.0027 - val_accuracy: 0.9856
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.9069e-04 - accuracy: 0.9919 - val_loss: 0.0023 - val_accuracy: 0.9764
2938/2938 [==============================] - 2s 505us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2295 - accuracy: 0.7788 - val_loss: 0.0011 - val_accuracy: 0.9789
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0015 - accuracy: 0.9900 - val_loss: 9.3789e-05 - val_accuracy: 0.9964
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.8081e-04 - accuracy: 0.9934 - val_loss: 0.0014 - val_accuracy: 0.9879
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.2028e-04 - accuracy: 0.9943 - val_loss: 3.9279e-06 - val_accuracy: 0.9992
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.5874e-04 - accuracy: 0.9937 - val_loss: 2.2210e-04 - val_accuracy: 0.9937
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.1237e-04 - accuracy: 0.9959 - val_loss: 9.2732e-04 - val_accuracy: 0.9873
Epoch 7/10
5964/5964 [==============================] - 11s 2ms/step - loss: 4.1026e-04 - accuracy: 0.9952 - val_loss: 0.0053 - val_accuracy: 0.9724
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0038 - accuracy: 0.9880 - val_loss: 1.8961e-05 - val_accuracy: 0.9982
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9931 - val_loss: 0.0036 - val_accuracy: 0.9844
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.0874e-04 - accuracy: 0.9923 - val_loss: 0.0023 - val_accuracy: 0.9795
2938/2938 [==============================] - 2s 563us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2205 - accuracy: 0.7721 - val_loss: 1.8490e-04 - val_accuracy: 0.9909
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0018 - accuracy: 0.9910 - val_loss: 0.0022 - val_accuracy: 0.9823
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0010 - accuracy: 0.9919 - val_loss: 0.0019 - val_accuracy: 0.9860
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.8025e-04 - accuracy: 0.9949 - val_loss: 3.5713e-06 - val_accuracy: 0.9993
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.5930e-04 - accuracy: 0.9932 - val_loss: 5.2104e-05 - val_accuracy: 0.9968
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.3828e-04 - accuracy: 0.9962 - val_loss: 0.0011 - val_accuracy: 0.9837
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.4112e-04 - accuracy: 0.9939 - val_loss: 0.0010 - val_accuracy: 0.9900
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9915 - val_loss: 2.7616e-05 - val_accuracy: 0.9978
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0018 - accuracy: 0.9920 - val_loss: 0.0024 - val_accuracy: 0.9811
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.9179e-04 - accuracy: 0.9921 - val_loss: 0.0022 - val_accuracy: 0.9783
2938/2938 [==============================] - 2s 508us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2146 - accuracy: 0.8073 - val_loss: 0.0041 - val_accuracy: 0.9638
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9911 - val_loss: 2.0278e-04 - val_accuracy: 0.9943
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0011 - accuracy: 0.9922 - val_loss: 8.4932e-04 - val_accuracy: 0.9894
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 5.8817e-04 - accuracy: 0.9940 - val_loss: 8.1787e-06 - val_accuracy: 0.9988
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.4110e-04 - accuracy: 0.9945 - val_loss: 5.9755e-05 - val_accuracy: 0.9966
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.5209e-04 - accuracy: 0.9960 - val_loss: 9.7591e-04 - val_accuracy: 0.9875
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.7731e-04 - accuracy: 0.9951 - val_loss: 0.0025 - val_accuracy: 0.9853
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9901 - val_loss: 1.2515e-05 - val_accuracy: 0.9988
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0016 - accuracy: 0.9940 - val_loss: 0.0038 - val_accuracy: 0.9830
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 6.9216e-04 - accuracy: 0.9927 - val_loss: 0.0051 - val_accuracy: 0.9692
2938/2938 [==============================] - 2s 506us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2006 - accuracy: 0.7866 - val_loss: 0.0018 - val_accuracy: 0.9726
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9903 - val_loss: 3.6475e-05 - val_accuracy: 0.9974
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 8.0648e-04 - accuracy: 0.9929 - val_loss: 3.2045e-04 - val_accuracy: 0.9940
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 4.5670e-04 - accuracy: 0.9945 - val_loss: 3.0967e-06 - val_accuracy: 0.9991
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 9.3938e-04 - accuracy: 0.9937 - val_loss: 1.0909e-04 - val_accuracy: 0.9950
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.3737e-04 - accuracy: 0.9959 - val_loss: 6.8811e-04 - val_accuracy: 0.9883
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.4069e-04 - accuracy: 0.9955 - val_loss: 0.0022 - val_accuracy: 0.9847
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0034 - accuracy: 0.9900 - val_loss: 2.1078e-04 - val_accuracy: 0.9942
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0018 - accuracy: 0.9938 - val_loss: 0.0011 - val_accuracy: 0.9897
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 6.8268e-04 - accuracy: 0.9926 - val_loss: 0.0031 - val_accuracy: 0.9738
2938/2938 [==============================] - 1s 489us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2184 - accuracy: 0.7877 - val_loss: 0.0075 - val_accuracy: 0.9587
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0013 - accuracy: 0.9912 - val_loss: 6.5324e-05 - val_accuracy: 0.9963
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 9.6679e-04 - accuracy: 0.9928 - val_loss: 5.4989e-04 - val_accuracy: 0.9914
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.0135e-04 - accuracy: 0.9929 - val_loss: 7.6205e-06 - val_accuracy: 0.9990
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 5.8127e-04 - accuracy: 0.9929 - val_loss: 5.5006e-05 - val_accuracy: 0.9960
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.5529e-04 - accuracy: 0.9960 - val_loss: 8.9819e-04 - val_accuracy: 0.9855
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 3.9595e-04 - accuracy: 0.9956 - val_loss: 0.0032 - val_accuracy: 0.9821
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0029 - accuracy: 0.9902 - val_loss: 1.4756e-04 - val_accuracy: 0.9956
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0015 - accuracy: 0.9925 - val_loss: 0.0021 - val_accuracy: 0.9845
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 7.3236e-04 - accuracy: 0.9922 - val_loss: 0.0027 - val_accuracy: 0.9694
2938/2938 [==============================] - 1s 484us/step
Mean average precision over 10 runs: 0.0362
Out[ ]:
[0.0226,
 0.0259,
 0.0469,
 0.0233,
 0.0467,
 0.0167,
 0.0349,
 0.0419,
 0.0373,
 0.0658]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0362
Coefficient of variation over 10 runs: 0.3885
Out[ ]:
[0.0226,
 0.0259,
 0.0469,
 0.0233,
 0.0467,
 0.0167,
 0.0349,
 0.0419,
 0.0373,
 0.0658]

Model Six¶

Two layer overcomplete autoencoder with linear activation and dropout¶

29 -> 40 -> 29¶

Dropout percentage: 10%¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    model.add(Dense(units=40, activation='linear',input_dim=29))
    model.add(Dropout(0.10))

    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2984 - accuracy: 0.6287 - val_loss: 0.0101 - val_accuracy: 0.9635
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0771 - accuracy: 0.7832 - val_loss: 0.0077 - val_accuracy: 0.9731
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0773 - accuracy: 0.7832 - val_loss: 0.0091 - val_accuracy: 0.9727
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0744 - accuracy: 0.7827 - val_loss: 0.0091 - val_accuracy: 0.9688
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0771 - accuracy: 0.7835 - val_loss: 0.0075 - val_accuracy: 0.9739
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0760 - accuracy: 0.7846 - val_loss: 0.0074 - val_accuracy: 0.9721
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0757 - accuracy: 0.7854 - val_loss: 0.0085 - val_accuracy: 0.9694
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0895 - accuracy: 0.7837 - val_loss: 0.0098 - val_accuracy: 0.9607
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0787 - accuracy: 0.7856 - val_loss: 0.0090 - val_accuracy: 0.9687
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7847 - val_loss: 0.0123 - val_accuracy: 0.9678
2938/2938 [==============================] - 2s 504us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3124 - accuracy: 0.6303 - val_loss: 0.0098 - val_accuracy: 0.9694
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0767 - accuracy: 0.7856 - val_loss: 0.0083 - val_accuracy: 0.9704
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7865 - val_loss: 0.0093 - val_accuracy: 0.9719
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0761 - accuracy: 0.7862 - val_loss: 0.0103 - val_accuracy: 0.9639
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0768 - accuracy: 0.7868 - val_loss: 0.0075 - val_accuracy: 0.9728
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0751 - accuracy: 0.7883 - val_loss: 0.0093 - val_accuracy: 0.9678
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0776 - accuracy: 0.7860 - val_loss: 0.0096 - val_accuracy: 0.9679
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0868 - accuracy: 0.7868 - val_loss: 0.0106 - val_accuracy: 0.9634
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0870 - accuracy: 0.7847 - val_loss: 0.0082 - val_accuracy: 0.9749
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0761 - accuracy: 0.7866 - val_loss: 0.0146 - val_accuracy: 0.9615
2938/2938 [==============================] - 1s 483us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3070 - accuracy: 0.6297 - val_loss: 0.0093 - val_accuracy: 0.9671
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0766 - accuracy: 0.7882 - val_loss: 0.0080 - val_accuracy: 0.9702
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0769 - accuracy: 0.7887 - val_loss: 0.0092 - val_accuracy: 0.9733
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0743 - accuracy: 0.7868 - val_loss: 0.0099 - val_accuracy: 0.9647
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0775 - accuracy: 0.7862 - val_loss: 0.0079 - val_accuracy: 0.9742
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0745 - accuracy: 0.7896 - val_loss: 0.0089 - val_accuracy: 0.9706
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0778 - accuracy: 0.7865 - val_loss: 0.0084 - val_accuracy: 0.9685
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0809 - accuracy: 0.7837 - val_loss: 0.0091 - val_accuracy: 0.9668
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0911 - accuracy: 0.7836 - val_loss: 0.0089 - val_accuracy: 0.9685
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0756 - accuracy: 0.7843 - val_loss: 0.0136 - val_accuracy: 0.9624
2938/2938 [==============================] - 1s 486us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3393 - accuracy: 0.6201 - val_loss: 0.0091 - val_accuracy: 0.9690
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0752 - accuracy: 0.7860 - val_loss: 0.0081 - val_accuracy: 0.9725
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0759 - accuracy: 0.7847 - val_loss: 0.0082 - val_accuracy: 0.9740
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0742 - accuracy: 0.7863 - val_loss: 0.0089 - val_accuracy: 0.9665
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0798 - accuracy: 0.7827 - val_loss: 0.0071 - val_accuracy: 0.9762
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0746 - accuracy: 0.7865 - val_loss: 0.0084 - val_accuracy: 0.9677
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0763 - accuracy: 0.7827 - val_loss: 0.0073 - val_accuracy: 0.9783
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0866 - accuracy: 0.7852 - val_loss: 0.0109 - val_accuracy: 0.9575
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0813 - accuracy: 0.7844 - val_loss: 0.0088 - val_accuracy: 0.9686
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0746 - accuracy: 0.7849 - val_loss: 0.0117 - val_accuracy: 0.9654
2938/2938 [==============================] - 1s 484us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3210 - accuracy: 0.6188 - val_loss: 0.0093 - val_accuracy: 0.9704
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0763 - accuracy: 0.7866 - val_loss: 0.0078 - val_accuracy: 0.9743
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0757 - accuracy: 0.7863 - val_loss: 0.0083 - val_accuracy: 0.9732
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0733 - accuracy: 0.7865 - val_loss: 0.0113 - val_accuracy: 0.9624
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0785 - accuracy: 0.7826 - val_loss: 0.0076 - val_accuracy: 0.9757
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0749 - accuracy: 0.7866 - val_loss: 0.0089 - val_accuracy: 0.9698
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0767 - accuracy: 0.7852 - val_loss: 0.0075 - val_accuracy: 0.9744
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0962 - accuracy: 0.7829 - val_loss: 0.0084 - val_accuracy: 0.9720
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0819 - accuracy: 0.7833 - val_loss: 0.0089 - val_accuracy: 0.9686
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0753 - accuracy: 0.7848 - val_loss: 0.0160 - val_accuracy: 0.9613
2938/2938 [==============================] - 1s 487us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3324 - accuracy: 0.6174 - val_loss: 0.0096 - val_accuracy: 0.9675
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0772 - accuracy: 0.7850 - val_loss: 0.0075 - val_accuracy: 0.9751
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0770 - accuracy: 0.7866 - val_loss: 0.0092 - val_accuracy: 0.9710
Epoch 4/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0745 - accuracy: 0.7875 - val_loss: 0.0093 - val_accuracy: 0.9684
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7855 - val_loss: 0.0071 - val_accuracy: 0.9755
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0743 - accuracy: 0.7873 - val_loss: 0.0096 - val_accuracy: 0.9660
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0771 - accuracy: 0.7836 - val_loss: 0.0081 - val_accuracy: 0.9723
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.1045 - accuracy: 0.7827 - val_loss: 0.0092 - val_accuracy: 0.9638
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0876 - accuracy: 0.7870 - val_loss: 0.0095 - val_accuracy: 0.9666
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0757 - accuracy: 0.7867 - val_loss: 0.0127 - val_accuracy: 0.9662
2938/2938 [==============================] - 1s 487us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3154 - accuracy: 0.6137 - val_loss: 0.0088 - val_accuracy: 0.9717
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0773 - accuracy: 0.7849 - val_loss: 0.0077 - val_accuracy: 0.9716
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0763 - accuracy: 0.7868 - val_loss: 0.0088 - val_accuracy: 0.9733
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0751 - accuracy: 0.7884 - val_loss: 0.0107 - val_accuracy: 0.9672
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0776 - accuracy: 0.7850 - val_loss: 0.0076 - val_accuracy: 0.9748
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0759 - accuracy: 0.7845 - val_loss: 0.0077 - val_accuracy: 0.9739
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0758 - accuracy: 0.7842 - val_loss: 0.0085 - val_accuracy: 0.9730
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0920 - accuracy: 0.7840 - val_loss: 0.0103 - val_accuracy: 0.9546
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0855 - accuracy: 0.7843 - val_loss: 0.0080 - val_accuracy: 0.9722
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0769 - accuracy: 0.7893 - val_loss: 0.0123 - val_accuracy: 0.9618
2938/2938 [==============================] - 1s 495us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3157 - accuracy: 0.6396 - val_loss: 0.0091 - val_accuracy: 0.9680
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0765 - accuracy: 0.7870 - val_loss: 0.0077 - val_accuracy: 0.9744
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7884 - val_loss: 0.0082 - val_accuracy: 0.9723
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0741 - accuracy: 0.7852 - val_loss: 0.0094 - val_accuracy: 0.9703
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0786 - accuracy: 0.7860 - val_loss: 0.0075 - val_accuracy: 0.9771
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0753 - accuracy: 0.7870 - val_loss: 0.0076 - val_accuracy: 0.9719
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0764 - accuracy: 0.7878 - val_loss: 0.0090 - val_accuracy: 0.9672
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0851 - accuracy: 0.7828 - val_loss: 0.0094 - val_accuracy: 0.9661
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0789 - accuracy: 0.7858 - val_loss: 0.0100 - val_accuracy: 0.9690
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0745 - accuracy: 0.7872 - val_loss: 0.0121 - val_accuracy: 0.9619
2938/2938 [==============================] - 1s 493us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3057 - accuracy: 0.6235 - val_loss: 0.0088 - val_accuracy: 0.9673
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0776 - accuracy: 0.7856 - val_loss: 0.0079 - val_accuracy: 0.9723
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0766 - accuracy: 0.7863 - val_loss: 0.0087 - val_accuracy: 0.9730
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0732 - accuracy: 0.7872 - val_loss: 0.0090 - val_accuracy: 0.9690
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0770 - accuracy: 0.7847 - val_loss: 0.0074 - val_accuracy: 0.9727
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0762 - accuracy: 0.7843 - val_loss: 0.0080 - val_accuracy: 0.9711
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0775 - accuracy: 0.7832 - val_loss: 0.0080 - val_accuracy: 0.9732
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0970 - accuracy: 0.7838 - val_loss: 0.0097 - val_accuracy: 0.9678
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0821 - accuracy: 0.7863 - val_loss: 0.0094 - val_accuracy: 0.9706
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0752 - accuracy: 0.7837 - val_loss: 0.0141 - val_accuracy: 0.9587
2938/2938 [==============================] - 2s 510us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3187 - accuracy: 0.6286 - val_loss: 0.0092 - val_accuracy: 0.9706
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0773 - accuracy: 0.7853 - val_loss: 0.0078 - val_accuracy: 0.9685
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0772 - accuracy: 0.7864 - val_loss: 0.0082 - val_accuracy: 0.9750
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0742 - accuracy: 0.7875 - val_loss: 0.0095 - val_accuracy: 0.9682
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0787 - accuracy: 0.7878 - val_loss: 0.0072 - val_accuracy: 0.9772
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0762 - accuracy: 0.7864 - val_loss: 0.0080 - val_accuracy: 0.9723
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0769 - accuracy: 0.7856 - val_loss: 0.0076 - val_accuracy: 0.9749
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0952 - accuracy: 0.7847 - val_loss: 0.0090 - val_accuracy: 0.9670
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0842 - accuracy: 0.7824 - val_loss: 0.0083 - val_accuracy: 0.9736
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0755 - accuracy: 0.7837 - val_loss: 0.0139 - val_accuracy: 0.9571
2938/2938 [==============================] - 2s 580us/step
Mean average precision over 10 runs: 0.1372
Out[ ]:
[0.1539,
 0.1048,
 0.0913,
 0.1385,
 0.1235,
 0.1244,
 0.2411,
 0.1668,
 0.1482,
 0.0798]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.1372
Coefficient of variation over 10 runs: 0.3167
Out[ ]:
[0.1539,
 0.1048,
 0.0913,
 0.1385,
 0.1235,
 0.1244,
 0.2411,
 0.1668,
 0.1482,
 0.0798]

Model Seven¶

Two layer sparse overcomplete autoencoder with linear activation¶

29 -> 40 -> 29¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    model.add(Dense(units=40, activation='linear',  \
        activity_regularizer=regularizers.l1(10e-5), input_dim=29))

    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2077 - accuracy: 0.7909 - val_loss: 0.0038 - val_accuracy: 0.9781
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0041 - accuracy: 0.9898 - val_loss: 0.0023 - val_accuracy: 0.9947
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0031 - accuracy: 0.9931 - val_loss: 0.0037 - val_accuracy: 0.9851
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9932 - val_loss: 0.0020 - val_accuracy: 0.9972
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9913 - val_loss: 0.0017 - val_accuracy: 0.9947
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9950 - val_loss: 0.0032 - val_accuracy: 0.9827
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9934 - val_loss: 0.0061 - val_accuracy: 0.9747
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0054 - accuracy: 0.9859 - val_loss: 0.0015 - val_accuracy: 0.9939
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0045 - accuracy: 0.9911 - val_loss: 0.0069 - val_accuracy: 0.9757
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0026 - accuracy: 0.9900 - val_loss: 0.0074 - val_accuracy: 0.9643
2938/2938 [==============================] - 1s 491us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2164 - accuracy: 0.7922 - val_loss: 0.0054 - val_accuracy: 0.9695
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0044 - accuracy: 0.9898 - val_loss: 0.0022 - val_accuracy: 0.9964
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9938 - val_loss: 0.0025 - val_accuracy: 0.9924
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0026 - accuracy: 0.9930 - val_loss: 0.0019 - val_accuracy: 0.9968
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9912 - val_loss: 0.0016 - val_accuracy: 0.9951
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9954 - val_loss: 0.0031 - val_accuracy: 0.9797
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9929 - val_loss: 0.0036 - val_accuracy: 0.9818
Epoch 8/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0046 - accuracy: 0.9881 - val_loss: 0.0014 - val_accuracy: 0.9981
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0041 - accuracy: 0.9914 - val_loss: 0.0079 - val_accuracy: 0.9769
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0025 - accuracy: 0.9895 - val_loss: 0.0076 - val_accuracy: 0.9602
2938/2938 [==============================] - 1s 494us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2097 - accuracy: 0.7888 - val_loss: 0.0045 - val_accuracy: 0.9725
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0040 - accuracy: 0.9907 - val_loss: 0.0024 - val_accuracy: 0.9921
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0028 - accuracy: 0.9925 - val_loss: 0.0027 - val_accuracy: 0.9889
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9939 - val_loss: 0.0017 - val_accuracy: 0.9989
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9929 - val_loss: 0.0016 - val_accuracy: 0.9967
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9953 - val_loss: 0.0035 - val_accuracy: 0.9798
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0022 - accuracy: 0.9927 - val_loss: 0.0049 - val_accuracy: 0.9792
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0049 - accuracy: 0.9876 - val_loss: 0.0013 - val_accuracy: 0.9984
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0034 - accuracy: 0.9915 - val_loss: 0.0067 - val_accuracy: 0.9771
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9896 - val_loss: 0.0040 - val_accuracy: 0.9731
2938/2938 [==============================] - 2s 495us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2410 - accuracy: 0.7792 - val_loss: 0.0138 - val_accuracy: 0.9481
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0039 - accuracy: 0.9897 - val_loss: 0.0022 - val_accuracy: 0.9960
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0028 - accuracy: 0.9927 - val_loss: 0.0021 - val_accuracy: 0.9950
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9947 - val_loss: 0.0022 - val_accuracy: 0.9946
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9903 - val_loss: 0.0016 - val_accuracy: 0.9964
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9951 - val_loss: 0.0034 - val_accuracy: 0.9780
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9929 - val_loss: 0.0041 - val_accuracy: 0.9797
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0046 - accuracy: 0.9877 - val_loss: 0.0013 - val_accuracy: 0.9980
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0035 - accuracy: 0.9907 - val_loss: 0.0075 - val_accuracy: 0.9703
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9888 - val_loss: 0.0055 - val_accuracy: 0.9705
2938/2938 [==============================] - 1s 485us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2206 - accuracy: 0.7812 - val_loss: 0.0058 - val_accuracy: 0.9632
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0039 - accuracy: 0.9902 - val_loss: 0.0022 - val_accuracy: 0.9964
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0029 - accuracy: 0.9925 - val_loss: 0.0038 - val_accuracy: 0.9843
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9928 - val_loss: 0.0016 - val_accuracy: 0.9989
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9913 - val_loss: 0.0016 - val_accuracy: 0.9951
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0019 - accuracy: 0.9950 - val_loss: 0.0030 - val_accuracy: 0.9800
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0021 - accuracy: 0.9930 - val_loss: 0.0029 - val_accuracy: 0.9856
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0046 - accuracy: 0.9874 - val_loss: 0.0014 - val_accuracy: 0.9960
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0045 - accuracy: 0.9911 - val_loss: 0.0066 - val_accuracy: 0.9775
Epoch 10/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0025 - accuracy: 0.9895 - val_loss: 0.0060 - val_accuracy: 0.9633
2938/2938 [==============================] - 1s 490us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2321 - accuracy: 0.7788 - val_loss: 0.0039 - val_accuracy: 0.9768
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0039 - accuracy: 0.9895 - val_loss: 0.0022 - val_accuracy: 0.9947
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0030 - accuracy: 0.9924 - val_loss: 0.0027 - val_accuracy: 0.9874
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9937 - val_loss: 0.0017 - val_accuracy: 0.9990
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9925 - val_loss: 0.0016 - val_accuracy: 0.9953
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0019 - accuracy: 0.9953 - val_loss: 0.0038 - val_accuracy: 0.9755
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9931 - val_loss: 0.0040 - val_accuracy: 0.9814
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0045 - accuracy: 0.9882 - val_loss: 0.0015 - val_accuracy: 0.9940
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0047 - accuracy: 0.9905 - val_loss: 0.0092 - val_accuracy: 0.9716
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9892 - val_loss: 0.0053 - val_accuracy: 0.9669
2938/2938 [==============================] - 1s 495us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2231 - accuracy: 0.7721 - val_loss: 0.0028 - val_accuracy: 0.9888
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0042 - accuracy: 0.9906 - val_loss: 0.0023 - val_accuracy: 0.9944
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0028 - accuracy: 0.9928 - val_loss: 0.0038 - val_accuracy: 0.9847
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9942 - val_loss: 0.0021 - val_accuracy: 0.9960
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9910 - val_loss: 0.0016 - val_accuracy: 0.9943
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9950 - val_loss: 0.0027 - val_accuracy: 0.9824
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9926 - val_loss: 0.0032 - val_accuracy: 0.9867
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0049 - accuracy: 0.9872 - val_loss: 0.0014 - val_accuracy: 0.9963
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0039 - accuracy: 0.9907 - val_loss: 0.0098 - val_accuracy: 0.9699
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0026 - accuracy: 0.9891 - val_loss: 0.0069 - val_accuracy: 0.9637
2938/2938 [==============================] - 1s 489us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2173 - accuracy: 0.8072 - val_loss: 0.0041 - val_accuracy: 0.9791
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0040 - accuracy: 0.9908 - val_loss: 0.0024 - val_accuracy: 0.9936
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0031 - accuracy: 0.9926 - val_loss: 0.0029 - val_accuracy: 0.9884
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0024 - accuracy: 0.9934 - val_loss: 0.0017 - val_accuracy: 0.9986
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0023 - accuracy: 0.9924 - val_loss: 0.0016 - val_accuracy: 0.9950
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9952 - val_loss: 0.0033 - val_accuracy: 0.9814
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0021 - accuracy: 0.9939 - val_loss: 0.0036 - val_accuracy: 0.9829
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0046 - accuracy: 0.9874 - val_loss: 0.0016 - val_accuracy: 0.9930
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0043 - accuracy: 0.9897 - val_loss: 0.0088 - val_accuracy: 0.9729
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9900 - val_loss: 0.0052 - val_accuracy: 0.9720
2938/2938 [==============================] - 1s 488us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2034 - accuracy: 0.7866 - val_loss: 0.0040 - val_accuracy: 0.9757
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0039 - accuracy: 0.9911 - val_loss: 0.0022 - val_accuracy: 0.9968
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0031 - accuracy: 0.9929 - val_loss: 0.0027 - val_accuracy: 0.9900
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0024 - accuracy: 0.9937 - val_loss: 0.0017 - val_accuracy: 0.9990
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9925 - val_loss: 0.0016 - val_accuracy: 0.9962
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9948 - val_loss: 0.0035 - val_accuracy: 0.9785
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0022 - accuracy: 0.9923 - val_loss: 0.0037 - val_accuracy: 0.9825
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0047 - accuracy: 0.9871 - val_loss: 0.0014 - val_accuracy: 0.9972
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0040 - accuracy: 0.9904 - val_loss: 0.0072 - val_accuracy: 0.9749
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0025 - accuracy: 0.9893 - val_loss: 0.0066 - val_accuracy: 0.9661
2938/2938 [==============================] - 1s 484us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2211 - accuracy: 0.7877 - val_loss: 0.0096 - val_accuracy: 0.9624
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0036 - accuracy: 0.9915 - val_loss: 0.0023 - val_accuracy: 0.9935
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0030 - accuracy: 0.9931 - val_loss: 0.0029 - val_accuracy: 0.9871
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9940 - val_loss: 0.0017 - val_accuracy: 0.9990
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0023 - accuracy: 0.9923 - val_loss: 0.0016 - val_accuracy: 0.9949
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9953 - val_loss: 0.0030 - val_accuracy: 0.9775
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0020 - accuracy: 0.9932 - val_loss: 0.0048 - val_accuracy: 0.9797
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0045 - accuracy: 0.9878 - val_loss: 0.0014 - val_accuracy: 0.9970
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0039 - accuracy: 0.9906 - val_loss: 0.0070 - val_accuracy: 0.9744
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0025 - accuracy: 0.9890 - val_loss: 0.0031 - val_accuracy: 0.9739
2938/2938 [==============================] - 1s 494us/step
Mean average precision over 10 runs: 0.0334
Out[ ]:
[0.0227, 0.0438, 0.0229, 0.0462, 0.0307, 0.0293, 0.042, 0.0296, 0.0367, 0.0303]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0334
Coefficient of variation over 10 runs: 0.2375
Out[ ]:
[0.0227, 0.0438, 0.0229, 0.0462, 0.0307, 0.0293, 0.042, 0.0296, 0.0367, 0.0303]

Model Eight¶

Two layer sparse overcomplete autoencoder with linear activation and dropout¶

29 -> 40 -> 29¶

Dropout percentage: 5%¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    model.add(Dense(units=40, activation='linear',  \
        activity_regularizer=regularizers.l1(10e-5), input_dim=29))
    model.add(Dropout(0.05))

    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE, y=X_train_AE,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()
    
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2588 - accuracy: 0.6891 - val_loss: 0.0079 - val_accuracy: 0.9656
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0412 - accuracy: 0.8589 - val_loss: 0.0051 - val_accuracy: 0.9793
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0398 - accuracy: 0.8616 - val_loss: 0.0061 - val_accuracy: 0.9774
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8590 - val_loss: 0.0058 - val_accuracy: 0.9779
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0405 - accuracy: 0.8593 - val_loss: 0.0043 - val_accuracy: 0.9799
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0399 - accuracy: 0.8601 - val_loss: 0.0052 - val_accuracy: 0.9767
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0394 - accuracy: 0.8627 - val_loss: 0.0050 - val_accuracy: 0.9758
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0450 - accuracy: 0.8597 - val_loss: 0.0074 - val_accuracy: 0.9649
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0428 - accuracy: 0.8590 - val_loss: 0.0053 - val_accuracy: 0.9781
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0407 - accuracy: 0.8629 - val_loss: 0.0123 - val_accuracy: 0.9602
2938/2938 [==============================] - 1s 485us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2650 - accuracy: 0.6930 - val_loss: 0.0080 - val_accuracy: 0.9709
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0406 - accuracy: 0.8611 - val_loss: 0.0061 - val_accuracy: 0.9735
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0404 - accuracy: 0.8626 - val_loss: 0.0066 - val_accuracy: 0.9779
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0404 - accuracy: 0.8616 - val_loss: 0.0069 - val_accuracy: 0.9719
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0410 - accuracy: 0.8605 - val_loss: 0.0043 - val_accuracy: 0.9830
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8632 - val_loss: 0.0060 - val_accuracy: 0.9714
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8628 - val_loss: 0.0105 - val_accuracy: 0.9612
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0460 - accuracy: 0.8603 - val_loss: 0.0099 - val_accuracy: 0.9652
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0529 - accuracy: 0.8602 - val_loss: 0.0050 - val_accuracy: 0.9816
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.8614 - val_loss: 0.0089 - val_accuracy: 0.9718
2938/2938 [==============================] - 2s 508us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2607 - accuracy: 0.6899 - val_loss: 0.0070 - val_accuracy: 0.9721
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0409 - accuracy: 0.8624 - val_loss: 0.0050 - val_accuracy: 0.9787
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0411 - accuracy: 0.8630 - val_loss: 0.0056 - val_accuracy: 0.9804
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0394 - accuracy: 0.8636 - val_loss: 0.0079 - val_accuracy: 0.9681
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0410 - accuracy: 0.8626 - val_loss: 0.0044 - val_accuracy: 0.9811
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8650 - val_loss: 0.0055 - val_accuracy: 0.9747
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.8627 - val_loss: 0.0056 - val_accuracy: 0.9761
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0436 - accuracy: 0.8599 - val_loss: 0.0067 - val_accuracy: 0.9693
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0507 - accuracy: 0.8610 - val_loss: 0.0057 - val_accuracy: 0.9758
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0384 - accuracy: 0.8628 - val_loss: 0.0057 - val_accuracy: 0.9781
2938/2938 [==============================] - 2s 524us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2940 - accuracy: 0.6803 - val_loss: 0.0072 - val_accuracy: 0.9722
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0401 - accuracy: 0.8616 - val_loss: 0.0058 - val_accuracy: 0.9759
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.8617 - val_loss: 0.0046 - val_accuracy: 0.9824
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0403 - accuracy: 0.8620 - val_loss: 0.0066 - val_accuracy: 0.9695
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0414 - accuracy: 0.8598 - val_loss: 0.0044 - val_accuracy: 0.9825
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0394 - accuracy: 0.8613 - val_loss: 0.0062 - val_accuracy: 0.9695
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0401 - accuracy: 0.8618 - val_loss: 0.0045 - val_accuracy: 0.9772
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0444 - accuracy: 0.8628 - val_loss: 0.0076 - val_accuracy: 0.9633
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8602 - val_loss: 0.0048 - val_accuracy: 0.9804
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0394 - accuracy: 0.8609 - val_loss: 0.0114 - val_accuracy: 0.9661
2938/2938 [==============================] - 1s 486us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2734 - accuracy: 0.6799 - val_loss: 0.0069 - val_accuracy: 0.9730
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.8605 - val_loss: 0.0047 - val_accuracy: 0.9819
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0411 - accuracy: 0.8602 - val_loss: 0.0061 - val_accuracy: 0.9763
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.8619 - val_loss: 0.0091 - val_accuracy: 0.9727
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8614 - val_loss: 0.0053 - val_accuracy: 0.9757
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0388 - accuracy: 0.8645 - val_loss: 0.0049 - val_accuracy: 0.9802
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0396 - accuracy: 0.8618 - val_loss: 0.0069 - val_accuracy: 0.9704
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0522 - accuracy: 0.8593 - val_loss: 0.0061 - val_accuracy: 0.9718
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0496 - accuracy: 0.8611 - val_loss: 0.0066 - val_accuracy: 0.9750
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8632 - val_loss: 0.0088 - val_accuracy: 0.9721
2938/2938 [==============================] - 1s 493us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2847 - accuracy: 0.6774 - val_loss: 0.0079 - val_accuracy: 0.9692
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0421 - accuracy: 0.8606 - val_loss: 0.0047 - val_accuracy: 0.9814
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.8628 - val_loss: 0.0052 - val_accuracy: 0.9784
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0395 - accuracy: 0.8626 - val_loss: 0.0061 - val_accuracy: 0.9715
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8619 - val_loss: 0.0041 - val_accuracy: 0.9845
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0396 - accuracy: 0.8624 - val_loss: 0.0073 - val_accuracy: 0.9717
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8627 - val_loss: 0.0047 - val_accuracy: 0.9811
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0570 - accuracy: 0.8607 - val_loss: 0.0065 - val_accuracy: 0.9683
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0453 - accuracy: 0.8624 - val_loss: 0.0047 - val_accuracy: 0.9804
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0398 - accuracy: 0.8623 - val_loss: 0.0084 - val_accuracy: 0.9695
2938/2938 [==============================] - 1s 484us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2723 - accuracy: 0.6726 - val_loss: 0.0067 - val_accuracy: 0.9733
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0417 - accuracy: 0.8606 - val_loss: 0.0048 - val_accuracy: 0.9803
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0404 - accuracy: 0.8627 - val_loss: 0.0052 - val_accuracy: 0.9783
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0392 - accuracy: 0.8631 - val_loss: 0.0080 - val_accuracy: 0.9717
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0399 - accuracy: 0.8623 - val_loss: 0.0046 - val_accuracy: 0.9789
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.8624 - val_loss: 0.0056 - val_accuracy: 0.9738
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0392 - accuracy: 0.8603 - val_loss: 0.0053 - val_accuracy: 0.9757
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0473 - accuracy: 0.8601 - val_loss: 0.0059 - val_accuracy: 0.9698
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0446 - accuracy: 0.8592 - val_loss: 0.0050 - val_accuracy: 0.9773
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8631 - val_loss: 0.0073 - val_accuracy: 0.9766
2938/2938 [==============================] - 2s 508us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2695 - accuracy: 0.7016 - val_loss: 0.0068 - val_accuracy: 0.9726
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0400 - accuracy: 0.8629 - val_loss: 0.0051 - val_accuracy: 0.9790
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8616 - val_loss: 0.0051 - val_accuracy: 0.9794
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0398 - accuracy: 0.8621 - val_loss: 0.0068 - val_accuracy: 0.9743
Epoch 5/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0420 - accuracy: 0.8609 - val_loss: 0.0041 - val_accuracy: 0.9842
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0392 - accuracy: 0.8639 - val_loss: 0.0055 - val_accuracy: 0.9759
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0389 - accuracy: 0.8646 - val_loss: 0.0059 - val_accuracy: 0.9740
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0482 - accuracy: 0.8617 - val_loss: 0.0068 - val_accuracy: 0.9698
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0438 - accuracy: 0.8617 - val_loss: 0.0063 - val_accuracy: 0.9749
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0393 - accuracy: 0.8631 - val_loss: 0.0079 - val_accuracy: 0.9765
2938/2938 [==============================] - 1s 488us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2599 - accuracy: 0.6842 - val_loss: 0.0072 - val_accuracy: 0.9710
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0419 - accuracy: 0.8613 - val_loss: 0.0052 - val_accuracy: 0.9771
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.8626 - val_loss: 0.0054 - val_accuracy: 0.9798
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8613 - val_loss: 0.0052 - val_accuracy: 0.9772
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8592 - val_loss: 0.0044 - val_accuracy: 0.9796
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0391 - accuracy: 0.8611 - val_loss: 0.0050 - val_accuracy: 0.9770
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0397 - accuracy: 0.8612 - val_loss: 0.0058 - val_accuracy: 0.9719
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0526 - accuracy: 0.8602 - val_loss: 0.0064 - val_accuracy: 0.9706
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0418 - accuracy: 0.8615 - val_loss: 0.0053 - val_accuracy: 0.9753
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0388 - accuracy: 0.8609 - val_loss: 0.0116 - val_accuracy: 0.9683
2938/2938 [==============================] - 2s 498us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2704 - accuracy: 0.6907 - val_loss: 0.0071 - val_accuracy: 0.9742
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0416 - accuracy: 0.8629 - val_loss: 0.0044 - val_accuracy: 0.9821
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0408 - accuracy: 0.8626 - val_loss: 0.0046 - val_accuracy: 0.9819
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0392 - accuracy: 0.8639 - val_loss: 0.0078 - val_accuracy: 0.9720
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0407 - accuracy: 0.8642 - val_loss: 0.0041 - val_accuracy: 0.9826
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0399 - accuracy: 0.8634 - val_loss: 0.0051 - val_accuracy: 0.9781
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0402 - accuracy: 0.8647 - val_loss: 0.0056 - val_accuracy: 0.9753
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0463 - accuracy: 0.8605 - val_loss: 0.0073 - val_accuracy: 0.9672
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0461 - accuracy: 0.8630 - val_loss: 0.0049 - val_accuracy: 0.9798
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0396 - accuracy: 0.8617 - val_loss: 0.0132 - val_accuracy: 0.9680
2938/2938 [==============================] - 2s 547us/step
Mean average precision over 10 runs: 0.0798
Out[ ]:
[0.0522, 0.0791, 0.0592, 0.0826, 0.0863, 0.0864, 0.076, 0.1203, 0.073, 0.0831]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0798
Coefficient of variation over 10 runs: 0.2169
Out[ ]:
[0.0522, 0.0791, 0.0592, 0.0826, 0.0863, 0.0864, 0.076, 0.1203, 0.073, 0.0831]

Model Nine¶

Two layer denoising undercomplete autoencoder with linear activation¶

29 -> 27 -> 29¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []

noise_factor = 0.50
X_train_AE_noisy = X_train_AE.copy() + noise_factor * \
 np.random.normal(loc=0.0, scale=1.0, size=X_train_AE.shape)
X_test_AE_noisy = X_test_AE.copy() + noise_factor * \
 np.random.normal(loc=0.0, scale=1.0, size=X_test_AE.shape) 
    
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Generate hidden layer with 27 nodes using linear activation
    model.add(Dense(units=27, activation='linear', input_dim=29))

    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE_noisy, y=X_train_AE_noisy,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test_AE_noisy, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()

print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3728 - accuracy: 0.6286 - val_loss: 0.0365 - val_accuracy: 0.8961
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.8654 - val_loss: 0.0362 - val_accuracy: 0.9092
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0575 - accuracy: 0.8698 - val_loss: 0.0356 - val_accuracy: 0.8923
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.8712 - val_loss: 0.0365 - val_accuracy: 0.9097
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.8823 - val_loss: 0.0362 - val_accuracy: 0.9192
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0544 - accuracy: 0.8865 - val_loss: 0.0372 - val_accuracy: 0.9112
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0525 - accuracy: 0.8893 - val_loss: 0.0366 - val_accuracy: 0.9406
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0669 - accuracy: 0.8968 - val_loss: 0.0369 - val_accuracy: 0.9551
Epoch 9/10
5964/5964 [==============================] - 9s 1ms/step - loss: 0.0743 - accuracy: 0.9008 - val_loss: 0.0390 - val_accuracy: 0.9480
Epoch 10/10
5964/5964 [==============================] - 9s 1ms/step - loss: 0.0503 - accuracy: 0.9078 - val_loss: 0.0384 - val_accuracy: 0.9456
2938/2938 [==============================] - 1s 482us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3704 - accuracy: 0.6403 - val_loss: 0.0365 - val_accuracy: 0.9104
Epoch 2/10
5964/5964 [==============================] - 9s 1ms/step - loss: 0.0573 - accuracy: 0.8636 - val_loss: 0.0361 - val_accuracy: 0.9319
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0591 - accuracy: 0.8747 - val_loss: 0.0358 - val_accuracy: 0.9499
Epoch 4/10
5964/5964 [==============================] - 9s 1ms/step - loss: 0.0524 - accuracy: 0.9036 - val_loss: 0.0368 - val_accuracy: 0.9374
Epoch 5/10
5964/5964 [==============================] - 9s 1ms/step - loss: 0.0538 - accuracy: 0.9004 - val_loss: 0.0357 - val_accuracy: 0.9457
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0524 - accuracy: 0.9029 - val_loss: 0.0367 - val_accuracy: 0.9404
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0540 - accuracy: 0.9029 - val_loss: 0.0370 - val_accuracy: 0.9480
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0589 - accuracy: 0.9025 - val_loss: 0.0380 - val_accuracy: 0.9523
Epoch 9/10
5964/5964 [==============================] - 9s 1ms/step - loss: 0.0650 - accuracy: 0.8979 - val_loss: 0.0371 - val_accuracy: 0.9577
Epoch 10/10
5964/5964 [==============================] - 9s 1ms/step - loss: 0.0506 - accuracy: 0.9087 - val_loss: 0.0375 - val_accuracy: 0.9570
2938/2938 [==============================] - 1s 482us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3890 - accuracy: 0.6440 - val_loss: 0.0365 - val_accuracy: 0.9264
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0565 - accuracy: 0.8926 - val_loss: 0.0362 - val_accuracy: 0.9441
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0610 - accuracy: 0.9030 - val_loss: 0.0359 - val_accuracy: 0.9599
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.9153 - val_loss: 0.0372 - val_accuracy: 0.9500
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0548 - accuracy: 0.9074 - val_loss: 0.0361 - val_accuracy: 0.9541
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0536 - accuracy: 0.9081 - val_loss: 0.0375 - val_accuracy: 0.9440
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0527 - accuracy: 0.9068 - val_loss: 0.0366 - val_accuracy: 0.9602
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0708 - accuracy: 0.9062 - val_loss: 0.0369 - val_accuracy: 0.9635
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0742 - accuracy: 0.9083 - val_loss: 0.0376 - val_accuracy: 0.9615
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0500 - accuracy: 0.9137 - val_loss: 0.0387 - val_accuracy: 0.9487
2938/2938 [==============================] - 2s 537us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3724 - accuracy: 0.6356 - val_loss: 0.0363 - val_accuracy: 0.9426
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0564 - accuracy: 0.8992 - val_loss: 0.0369 - val_accuracy: 0.9578
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0645 - accuracy: 0.9018 - val_loss: 0.0369 - val_accuracy: 0.9640
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0544 - accuracy: 0.9136 - val_loss: 0.0382 - val_accuracy: 0.9544
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.9108 - val_loss: 0.0364 - val_accuracy: 0.9679
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0535 - accuracy: 0.9121 - val_loss: 0.0371 - val_accuracy: 0.9533
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0524 - accuracy: 0.9113 - val_loss: 0.0370 - val_accuracy: 0.9595
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0713 - accuracy: 0.9055 - val_loss: 0.0369 - val_accuracy: 0.9658
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0754 - accuracy: 0.9138 - val_loss: 0.0378 - val_accuracy: 0.9631
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0499 - accuracy: 0.9124 - val_loss: 0.0372 - val_accuracy: 0.9607
2938/2938 [==============================] - 2s 510us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3878 - accuracy: 0.6345 - val_loss: 0.0369 - val_accuracy: 0.9202
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0572 - accuracy: 0.8875 - val_loss: 0.0371 - val_accuracy: 0.9387
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0621 - accuracy: 0.8984 - val_loss: 0.0366 - val_accuracy: 0.9496
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0557 - accuracy: 0.9120 - val_loss: 0.0379 - val_accuracy: 0.9516
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0554 - accuracy: 0.9090 - val_loss: 0.0365 - val_accuracy: 0.9612
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0538 - accuracy: 0.9080 - val_loss: 0.0372 - val_accuracy: 0.9482
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0521 - accuracy: 0.9065 - val_loss: 0.0369 - val_accuracy: 0.9546
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0716 - accuracy: 0.9013 - val_loss: 0.0370 - val_accuracy: 0.9640
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0741 - accuracy: 0.9104 - val_loss: 0.0374 - val_accuracy: 0.9626
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0500 - accuracy: 0.9127 - val_loss: 0.0395 - val_accuracy: 0.9463
2938/2938 [==============================] - 2s 506us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3700 - accuracy: 0.6312 - val_loss: 0.0374 - val_accuracy: 0.8904
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.8438 - val_loss: 0.0362 - val_accuracy: 0.8682
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0559 - accuracy: 0.8597 - val_loss: 0.0354 - val_accuracy: 0.9190
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0538 - accuracy: 0.8852 - val_loss: 0.0363 - val_accuracy: 0.9333
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0566 - accuracy: 0.8913 - val_loss: 0.0354 - val_accuracy: 0.9499
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0525 - accuracy: 0.8981 - val_loss: 0.0361 - val_accuracy: 0.9404
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0531 - accuracy: 0.8991 - val_loss: 0.0362 - val_accuracy: 0.9511
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0659 - accuracy: 0.8971 - val_loss: 0.0368 - val_accuracy: 0.9555
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0691 - accuracy: 0.9051 - val_loss: 0.0372 - val_accuracy: 0.9546
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0502 - accuracy: 0.9095 - val_loss: 0.0378 - val_accuracy: 0.9558
2938/2938 [==============================] - 2s 517us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3858 - accuracy: 0.6415 - val_loss: 0.0365 - val_accuracy: 0.8895
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0552 - accuracy: 0.8550 - val_loss: 0.0357 - val_accuracy: 0.8995
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0558 - accuracy: 0.8622 - val_loss: 0.0358 - val_accuracy: 0.8978
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0536 - accuracy: 0.8653 - val_loss: 0.0363 - val_accuracy: 0.9072
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0541 - accuracy: 0.8723 - val_loss: 0.0357 - val_accuracy: 0.9082
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0534 - accuracy: 0.8799 - val_loss: 0.0359 - val_accuracy: 0.9071
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0537 - accuracy: 0.8739 - val_loss: 0.0374 - val_accuracy: 0.9171
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0575 - accuracy: 0.8798 - val_loss: 0.0365 - val_accuracy: 0.9453
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0607 - accuracy: 0.9001 - val_loss: 0.0371 - val_accuracy: 0.9421
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0521 - accuracy: 0.9025 - val_loss: 0.0374 - val_accuracy: 0.9420
2938/2938 [==============================] - 2s 508us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.4367 - accuracy: 0.6155 - val_loss: 0.0363 - val_accuracy: 0.9223
Epoch 2/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0544 - accuracy: 0.8897 - val_loss: 0.0354 - val_accuracy: 0.9460
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0534 - accuracy: 0.9031 - val_loss: 0.0353 - val_accuracy: 0.9479
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0541 - accuracy: 0.9067 - val_loss: 0.0359 - val_accuracy: 0.9508
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0556 - accuracy: 0.9073 - val_loss: 0.0356 - val_accuracy: 0.9597
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0529 - accuracy: 0.9133 - val_loss: 0.0357 - val_accuracy: 0.9563
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0546 - accuracy: 0.9143 - val_loss: 0.0367 - val_accuracy: 0.9591
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0616 - accuracy: 0.9162 - val_loss: 0.0363 - val_accuracy: 0.9618
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0607 - accuracy: 0.9210 - val_loss: 0.0374 - val_accuracy: 0.9583
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0512 - accuracy: 0.9108 - val_loss: 0.0369 - val_accuracy: 0.9585
2938/2938 [==============================] - 2s 504us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3548 - accuracy: 0.6500 - val_loss: 0.0367 - val_accuracy: 0.9378
Epoch 2/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0565 - accuracy: 0.9002 - val_loss: 0.0369 - val_accuracy: 0.9587
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0657 - accuracy: 0.9036 - val_loss: 0.0371 - val_accuracy: 0.9642
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0552 - accuracy: 0.9156 - val_loss: 0.0377 - val_accuracy: 0.9593
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0555 - accuracy: 0.9130 - val_loss: 0.0365 - val_accuracy: 0.9689
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0535 - accuracy: 0.9129 - val_loss: 0.0370 - val_accuracy: 0.9566
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0521 - accuracy: 0.9115 - val_loss: 0.0367 - val_accuracy: 0.9597
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0712 - accuracy: 0.9032 - val_loss: 0.0368 - val_accuracy: 0.9647
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0744 - accuracy: 0.9106 - val_loss: 0.0376 - val_accuracy: 0.9616
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0500 - accuracy: 0.9132 - val_loss: 0.0377 - val_accuracy: 0.9561
2938/2938 [==============================] - 2s 501us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3874 - accuracy: 0.6347 - val_loss: 0.0371 - val_accuracy: 0.9209
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0567 - accuracy: 0.8791 - val_loss: 0.0367 - val_accuracy: 0.9159
Epoch 3/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0582 - accuracy: 0.8769 - val_loss: 0.0361 - val_accuracy: 0.9112
Epoch 4/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0554 - accuracy: 0.8828 - val_loss: 0.0382 - val_accuracy: 0.9209
Epoch 5/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0554 - accuracy: 0.8901 - val_loss: 0.0363 - val_accuracy: 0.9506
Epoch 6/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0537 - accuracy: 0.9039 - val_loss: 0.0382 - val_accuracy: 0.9413
Epoch 7/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0527 - accuracy: 0.9042 - val_loss: 0.0375 - val_accuracy: 0.9553
Epoch 8/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0709 - accuracy: 0.9023 - val_loss: 0.0369 - val_accuracy: 0.9630
Epoch 9/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0744 - accuracy: 0.9081 - val_loss: 0.0377 - val_accuracy: 0.9600
Epoch 10/10
5964/5964 [==============================] - 9s 2ms/step - loss: 0.0501 - accuracy: 0.9111 - val_loss: 0.0376 - val_accuracy: 0.9540
2938/2938 [==============================] - 2s 502us/step
Mean average precision over 10 runs: 0.0131
Out[ ]:
[0.0148, 0.0279, 0.0106, 0.007, 0.0117, 0.0159, 0.02, 0.0068, 0.0092, 0.0071]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0131
Coefficient of variation over 10 runs: 0.4922
Out[ ]:
[0.0148, 0.0279, 0.0106, 0.007, 0.0117, 0.0159, 0.02, 0.0068, 0.0092, 0.0071]

Model Ten¶

Two layer denoising overcomplete autoencoder with linear activation¶

And sparsity regularizer and dropout¶

29 -> 40 -> 29¶

Dropout percentage: 5%¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [ ]:
# 10 runs - We will capture mean of average precision
test_scores = []

noise_factor = 0.50
X_train_AE_noisy = X_train_AE.copy() + noise_factor * \
 np.random.normal(loc=0.0, scale=1.0, size=X_train_AE.shape)
X_test_AE_noisy = X_test_AE.copy() + noise_factor * \
 np.random.normal(loc=0.0, scale=1.0, size=X_test_AE.shape) 
    
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Generate hidden layer with 40 nodes using linear activation
    model.add(Dense(units=40, activation='linear',  \
        activity_regularizer=regularizers.l1(10e-5), input_dim=29))
    model.add(Dropout(0.05))

    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE_noisy, y=X_train_AE_noisy,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test_AE_noisy, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()
    
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2963 - accuracy: 0.6827 - val_loss: 0.0062 - val_accuracy: 0.9705
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8486 - val_loss: 0.0049 - val_accuracy: 0.9806
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8456 - val_loss: 0.0053 - val_accuracy: 0.9807
Epoch 4/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0489 - accuracy: 0.8468 - val_loss: 0.0050 - val_accuracy: 0.9807
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0502 - accuracy: 0.8471 - val_loss: 0.0041 - val_accuracy: 0.9803
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0499 - accuracy: 0.8478 - val_loss: 0.0044 - val_accuracy: 0.9786
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8485 - val_loss: 0.0045 - val_accuracy: 0.9777
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0567 - accuracy: 0.8471 - val_loss: 0.0065 - val_accuracy: 0.9686
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0539 - accuracy: 0.8452 - val_loss: 0.0047 - val_accuracy: 0.9803
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.8469 - val_loss: 0.0084 - val_accuracy: 0.9671
2938/2938 [==============================] - 2s 498us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3040 - accuracy: 0.6863 - val_loss: 0.0064 - val_accuracy: 0.9731
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0502 - accuracy: 0.8487 - val_loss: 0.0060 - val_accuracy: 0.9738
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8494 - val_loss: 0.0055 - val_accuracy: 0.9787
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0500 - accuracy: 0.8475 - val_loss: 0.0059 - val_accuracy: 0.9749
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8485 - val_loss: 0.0039 - val_accuracy: 0.9843
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8482 - val_loss: 0.0051 - val_accuracy: 0.9750
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8484 - val_loss: 0.0090 - val_accuracy: 0.9655
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0564 - accuracy: 0.8496 - val_loss: 0.0079 - val_accuracy: 0.9699
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0612 - accuracy: 0.8461 - val_loss: 0.0045 - val_accuracy: 0.9829
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8471 - val_loss: 0.0054 - val_accuracy: 0.9781
2938/2938 [==============================] - 2s 505us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.2983 - accuracy: 0.6837 - val_loss: 0.0054 - val_accuracy: 0.9781
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0504 - accuracy: 0.8485 - val_loss: 0.0047 - val_accuracy: 0.9787
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8487 - val_loss: 0.0046 - val_accuracy: 0.9824
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8493 - val_loss: 0.0067 - val_accuracy: 0.9716
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8478 - val_loss: 0.0041 - val_accuracy: 0.9822
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8507 - val_loss: 0.0048 - val_accuracy: 0.9758
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8496 - val_loss: 0.0047 - val_accuracy: 0.9796
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0532 - accuracy: 0.8465 - val_loss: 0.0062 - val_accuracy: 0.9725
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0606 - accuracy: 0.8474 - val_loss: 0.0054 - val_accuracy: 0.9762
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0483 - accuracy: 0.8491 - val_loss: 0.0050 - val_accuracy: 0.9782
2938/2938 [==============================] - 2s 496us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3338 - accuracy: 0.6718 - val_loss: 0.0058 - val_accuracy: 0.9746
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0496 - accuracy: 0.8503 - val_loss: 0.0055 - val_accuracy: 0.9764
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8476 - val_loss: 0.0042 - val_accuracy: 0.9841
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8474 - val_loss: 0.0058 - val_accuracy: 0.9726
Epoch 5/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0508 - accuracy: 0.8491 - val_loss: 0.0042 - val_accuracy: 0.9829
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8474 - val_loss: 0.0055 - val_accuracy: 0.9700
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8476 - val_loss: 0.0042 - val_accuracy: 0.9810
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0540 - accuracy: 0.8457 - val_loss: 0.0066 - val_accuracy: 0.9672
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0582 - accuracy: 0.8461 - val_loss: 0.0046 - val_accuracy: 0.9815
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8470 - val_loss: 0.0077 - val_accuracy: 0.9732
2938/2938 [==============================] - 2s 507us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3136 - accuracy: 0.6768 - val_loss: 0.0055 - val_accuracy: 0.9767
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.8476 - val_loss: 0.0043 - val_accuracy: 0.9822
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0503 - accuracy: 0.8481 - val_loss: 0.0050 - val_accuracy: 0.9787
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0486 - accuracy: 0.8493 - val_loss: 0.0072 - val_accuracy: 0.9763
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8493 - val_loss: 0.0046 - val_accuracy: 0.9776
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8478 - val_loss: 0.0043 - val_accuracy: 0.9819
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8500 - val_loss: 0.0061 - val_accuracy: 0.9735
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0616 - accuracy: 0.8471 - val_loss: 0.0049 - val_accuracy: 0.9761
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0587 - accuracy: 0.8490 - val_loss: 0.0064 - val_accuracy: 0.9750
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8490 - val_loss: 0.0050 - val_accuracy: 0.9806
2938/2938 [==============================] - 2s 501us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3261 - accuracy: 0.6746 - val_loss: 0.0059 - val_accuracy: 0.9734
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0514 - accuracy: 0.8482 - val_loss: 0.0044 - val_accuracy: 0.9813
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8494 - val_loss: 0.0045 - val_accuracy: 0.9817
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8488 - val_loss: 0.0052 - val_accuracy: 0.9743
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0494 - accuracy: 0.8496 - val_loss: 0.0039 - val_accuracy: 0.9844
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0493 - accuracy: 0.8498 - val_loss: 0.0059 - val_accuracy: 0.9753
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8486 - val_loss: 0.0046 - val_accuracy: 0.9805
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0678 - accuracy: 0.8490 - val_loss: 0.0062 - val_accuracy: 0.9726
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0554 - accuracy: 0.8472 - val_loss: 0.0040 - val_accuracy: 0.9838
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0498 - accuracy: 0.8472 - val_loss: 0.0058 - val_accuracy: 0.9763
2938/2938 [==============================] - 2s 510us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.3120 - accuracy: 0.6737 - val_loss: 0.0055 - val_accuracy: 0.9749
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0512 - accuracy: 0.8498 - val_loss: 0.0044 - val_accuracy: 0.9818
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0501 - accuracy: 0.8491 - val_loss: 0.0043 - val_accuracy: 0.9812
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8498 - val_loss: 0.0069 - val_accuracy: 0.9743
Epoch 5/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0497 - accuracy: 0.8485 - val_loss: 0.0043 - val_accuracy: 0.9814
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0498 - accuracy: 0.8495 - val_loss: 0.0048 - val_accuracy: 0.9757
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0489 - accuracy: 0.8485 - val_loss: 0.0044 - val_accuracy: 0.9789
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0572 - accuracy: 0.8478 - val_loss: 0.0055 - val_accuracy: 0.9702
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0543 - accuracy: 0.8465 - val_loss: 0.0040 - val_accuracy: 0.9823
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8490 - val_loss: 0.0054 - val_accuracy: 0.9795
2938/2938 [==============================] - 2s 506us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.3053 - accuracy: 0.6925 - val_loss: 0.0054 - val_accuracy: 0.9751
Epoch 2/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0494 - accuracy: 0.8490 - val_loss: 0.0047 - val_accuracy: 0.9802
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0496 - accuracy: 0.8483 - val_loss: 0.0044 - val_accuracy: 0.9819
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8471 - val_loss: 0.0056 - val_accuracy: 0.9782
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0519 - accuracy: 0.8466 - val_loss: 0.0038 - val_accuracy: 0.9854
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8516 - val_loss: 0.0046 - val_accuracy: 0.9781
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0487 - accuracy: 0.8492 - val_loss: 0.0055 - val_accuracy: 0.9777
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0575 - accuracy: 0.8501 - val_loss: 0.0058 - val_accuracy: 0.9734
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0533 - accuracy: 0.8479 - val_loss: 0.0059 - val_accuracy: 0.9764
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0491 - accuracy: 0.8489 - val_loss: 0.0056 - val_accuracy: 0.9786
2938/2938 [==============================] - 2s 509us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.2979 - accuracy: 0.6819 - val_loss: 0.0058 - val_accuracy: 0.9726
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0512 - accuracy: 0.8480 - val_loss: 0.0047 - val_accuracy: 0.9785
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8479 - val_loss: 0.0048 - val_accuracy: 0.9809
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0490 - accuracy: 0.8472 - val_loss: 0.0046 - val_accuracy: 0.9799
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0508 - accuracy: 0.8493 - val_loss: 0.0041 - val_accuracy: 0.9809
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0492 - accuracy: 0.8489 - val_loss: 0.0043 - val_accuracy: 0.9799
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0495 - accuracy: 0.8460 - val_loss: 0.0049 - val_accuracy: 0.9774
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0622 - accuracy: 0.8471 - val_loss: 0.0058 - val_accuracy: 0.9731
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0513 - accuracy: 0.8476 - val_loss: 0.0047 - val_accuracy: 0.9777
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8480 - val_loss: 0.0074 - val_accuracy: 0.9745
2938/2938 [==============================] - 2s 495us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.3105 - accuracy: 0.6789 - val_loss: 0.0058 - val_accuracy: 0.9769
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0511 - accuracy: 0.8462 - val_loss: 0.0041 - val_accuracy: 0.9823
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0505 - accuracy: 0.8490 - val_loss: 0.0041 - val_accuracy: 0.9837
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0488 - accuracy: 0.8497 - val_loss: 0.0066 - val_accuracy: 0.9733
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0506 - accuracy: 0.8488 - val_loss: 0.0038 - val_accuracy: 0.9819
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.0498 - accuracy: 0.8486 - val_loss: 0.0047 - val_accuracy: 0.9784
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8517 - val_loss: 0.0048 - val_accuracy: 0.9786
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0550 - accuracy: 0.8464 - val_loss: 0.0062 - val_accuracy: 0.9722
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0563 - accuracy: 0.8481 - val_loss: 0.0040 - val_accuracy: 0.9835
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.0497 - accuracy: 0.8478 - val_loss: 0.0078 - val_accuracy: 0.9733
2938/2938 [==============================] - 2s 501us/step
Mean average precision over 10 runs: 0.0239
Out[ ]:
[0.0155, 0.0307, 0.0122, 0.025, 0.0202, 0.02, 0.0257, 0.0244, 0.0343, 0.0308]

Results¶

In [ ]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0239
Coefficient of variation over 10 runs: 0.2791
Out[ ]:
[0.0155, 0.0307, 0.0122, 0.025, 0.0202, 0.02, 0.0257, 0.0244, 0.0343, 0.0308]

Model Eleven¶

Two layer denoising overcomplete autoencoder with ReLU activation¶

And sparsity regularizer and dropout¶

29 -> 40 -> 29¶

Dropout percentage: 5%¶

In [ ]:
tf.random.set_seed(42)
np.random.seed(42)
In [51]:
# 10 runs - We will capture mean of average precision
test_scores = []

noise_factor = 0.50
X_train_AE_noisy = X_train_AE.copy() + noise_factor * \
 np.random.normal(loc=0.0, scale=1.0, size=X_train_AE.shape)
X_test_AE_noisy = X_test_AE.copy() + noise_factor * \
 np.random.normal(loc=0.0, scale=1.0, size=X_test_AE.shape) 
    
for i in range(0,10):
    # Call neural network API
    model = Sequential()

    # Generate hidden layer with 40 nodes using linear activation
    model.add(Dense(units=40, activation='relu',  \
        activity_regularizer=regularizers.l1(10e-5), input_dim=29))
    model.add(Dropout(0.05))

    # Generate output layer with 29 nodes
    model.add(Dense(units=29, activation='linear'))

    # Compile the model
    model.compile(optimizer='adam',
                  loss='mean_squared_error',
                  metrics=['accuracy'])

    # Train the model
    num_epochs = 10
    batch_size = 32

    history = model.fit(x=X_train_AE_noisy, y=X_train_AE_noisy,
                        epochs=num_epochs,
                        batch_size=batch_size,
                        shuffle=True,
                        validation_data=(X_train_AE, X_train_AE),
                        verbose=1)

    # Evaluate on test set
    predictions = model.predict(X_test_AE_noisy, verbose=1)
    anomalyScoresAE = anomalyScores(X_test, predictions)
    preds, avgPrecision = plotResults(y_test, anomalyScoresAE, True)
    test_scores.append(avgPrecision)
    model.reset_states()
    
print(f"Mean average precision over 10 runs: {round(np.mean(test_scores),4)}")
[round(x,4) for x in test_scores]
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.4365 - accuracy: 0.5372 - val_loss: 0.0228 - val_accuracy: 0.9172
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1360 - accuracy: 0.7204 - val_loss: 0.0184 - val_accuracy: 0.9130
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1267 - accuracy: 0.7291 - val_loss: 0.0152 - val_accuracy: 0.9225
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1177 - accuracy: 0.7337 - val_loss: 0.0155 - val_accuracy: 0.9174
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1180 - accuracy: 0.7330 - val_loss: 0.0128 - val_accuracy: 0.9232
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1175 - accuracy: 0.7349 - val_loss: 0.0135 - val_accuracy: 0.9229
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1147 - accuracy: 0.7363 - val_loss: 0.0128 - val_accuracy: 0.9234
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1165 - accuracy: 0.7366 - val_loss: 0.0141 - val_accuracy: 0.9201
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1158 - accuracy: 0.7390 - val_loss: 0.0125 - val_accuracy: 0.9236
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1092 - accuracy: 0.7387 - val_loss: 0.0141 - val_accuracy: 0.9229
2938/2938 [==============================] - 2s 499us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.4213 - accuracy: 0.5432 - val_loss: 0.0220 - val_accuracy: 0.9098
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1317 - accuracy: 0.7235 - val_loss: 0.0186 - val_accuracy: 0.9203
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1282 - accuracy: 0.7284 - val_loss: 0.0154 - val_accuracy: 0.9275
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1204 - accuracy: 0.7328 - val_loss: 0.0159 - val_accuracy: 0.9176
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1181 - accuracy: 0.7377 - val_loss: 0.0126 - val_accuracy: 0.9219
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1171 - accuracy: 0.7404 - val_loss: 0.0147 - val_accuracy: 0.9179
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1166 - accuracy: 0.7399 - val_loss: 0.0127 - val_accuracy: 0.9229
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1165 - accuracy: 0.7415 - val_loss: 0.0126 - val_accuracy: 0.9237
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1143 - accuracy: 0.7403 - val_loss: 0.0127 - val_accuracy: 0.9255
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1142 - accuracy: 0.7433 - val_loss: 0.0149 - val_accuracy: 0.9240
2938/2938 [==============================] - 2s 495us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.4398 - accuracy: 0.5334 - val_loss: 0.0231 - val_accuracy: 0.9095
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1333 - accuracy: 0.7188 - val_loss: 0.0177 - val_accuracy: 0.9130
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1271 - accuracy: 0.7283 - val_loss: 0.0145 - val_accuracy: 0.9233
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1219 - accuracy: 0.7308 - val_loss: 0.0171 - val_accuracy: 0.9126
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1215 - accuracy: 0.7351 - val_loss: 0.0131 - val_accuracy: 0.9226
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1133 - accuracy: 0.7375 - val_loss: 0.0128 - val_accuracy: 0.9224
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1126 - accuracy: 0.7370 - val_loss: 0.0128 - val_accuracy: 0.9243
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1117 - accuracy: 0.7381 - val_loss: 0.0147 - val_accuracy: 0.9177
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1236 - accuracy: 0.7340 - val_loss: 0.0127 - val_accuracy: 0.9268
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1107 - accuracy: 0.7388 - val_loss: 0.0137 - val_accuracy: 0.9253
2938/2938 [==============================] - 2s 548us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.4654 - accuracy: 0.5268 - val_loss: 0.0241 - val_accuracy: 0.9015
Epoch 2/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1316 - accuracy: 0.7230 - val_loss: 0.0168 - val_accuracy: 0.9214
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1252 - accuracy: 0.7312 - val_loss: 0.0143 - val_accuracy: 0.9240
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1198 - accuracy: 0.7346 - val_loss: 0.0152 - val_accuracy: 0.9199
Epoch 5/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1182 - accuracy: 0.7351 - val_loss: 0.0124 - val_accuracy: 0.9256
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1150 - accuracy: 0.7383 - val_loss: 0.0137 - val_accuracy: 0.9140
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1138 - accuracy: 0.7373 - val_loss: 0.0135 - val_accuracy: 0.9192
Epoch 8/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1230 - accuracy: 0.7372 - val_loss: 0.0140 - val_accuracy: 0.9219
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1156 - accuracy: 0.7355 - val_loss: 0.0141 - val_accuracy: 0.9177
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1102 - accuracy: 0.7374 - val_loss: 0.0154 - val_accuracy: 0.9208
2938/2938 [==============================] - 2s 498us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.4246 - accuracy: 0.5415 - val_loss: 0.0222 - val_accuracy: 0.9146
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1308 - accuracy: 0.7255 - val_loss: 0.0178 - val_accuracy: 0.9193
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1229 - accuracy: 0.7305 - val_loss: 0.0134 - val_accuracy: 0.9281
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1172 - accuracy: 0.7352 - val_loss: 0.0155 - val_accuracy: 0.9184
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1176 - accuracy: 0.7391 - val_loss: 0.0123 - val_accuracy: 0.9260
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1116 - accuracy: 0.7433 - val_loss: 0.0124 - val_accuracy: 0.9214
Epoch 7/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1121 - accuracy: 0.7393 - val_loss: 0.0152 - val_accuracy: 0.9163
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1177 - accuracy: 0.7389 - val_loss: 0.0140 - val_accuracy: 0.9178
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1227 - accuracy: 0.7377 - val_loss: 0.0124 - val_accuracy: 0.9218
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1111 - accuracy: 0.7418 - val_loss: 0.0134 - val_accuracy: 0.9226
2938/2938 [==============================] - 2s 507us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.4539 - accuracy: 0.5296 - val_loss: 0.0256 - val_accuracy: 0.9088
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1347 - accuracy: 0.7207 - val_loss: 0.0179 - val_accuracy: 0.9229
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1274 - accuracy: 0.7280 - val_loss: 0.0158 - val_accuracy: 0.9200
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1189 - accuracy: 0.7341 - val_loss: 0.0162 - val_accuracy: 0.9179
Epoch 5/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1166 - accuracy: 0.7351 - val_loss: 0.0126 - val_accuracy: 0.9236
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1148 - accuracy: 0.7355 - val_loss: 0.0135 - val_accuracy: 0.9213
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1154 - accuracy: 0.7379 - val_loss: 0.0129 - val_accuracy: 0.9192
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1228 - accuracy: 0.7344 - val_loss: 0.0137 - val_accuracy: 0.9148
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1364 - accuracy: 0.7366 - val_loss: 0.0122 - val_accuracy: 0.9190
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1100 - accuracy: 0.7373 - val_loss: 0.0133 - val_accuracy: 0.9181
2938/2938 [==============================] - 1s 492us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.4422 - accuracy: 0.5274 - val_loss: 0.0229 - val_accuracy: 0.9164
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1381 - accuracy: 0.7222 - val_loss: 0.0196 - val_accuracy: 0.9217
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1273 - accuracy: 0.7277 - val_loss: 0.0159 - val_accuracy: 0.9234
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1205 - accuracy: 0.7330 - val_loss: 0.0157 - val_accuracy: 0.9190
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1195 - accuracy: 0.7345 - val_loss: 0.0137 - val_accuracy: 0.9241
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1149 - accuracy: 0.7370 - val_loss: 0.0137 - val_accuracy: 0.9221
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1141 - accuracy: 0.7380 - val_loss: 0.0130 - val_accuracy: 0.9220
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1130 - accuracy: 0.7382 - val_loss: 0.0136 - val_accuracy: 0.9168
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1128 - accuracy: 0.7384 - val_loss: 0.0122 - val_accuracy: 0.9205
Epoch 10/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1126 - accuracy: 0.7381 - val_loss: 0.0135 - val_accuracy: 0.9248
2938/2938 [==============================] - 2s 559us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.4188 - accuracy: 0.5494 - val_loss: 0.0238 - val_accuracy: 0.9029
Epoch 2/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1334 - accuracy: 0.7220 - val_loss: 0.0183 - val_accuracy: 0.9119
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1262 - accuracy: 0.7252 - val_loss: 0.0162 - val_accuracy: 0.9178
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1202 - accuracy: 0.7299 - val_loss: 0.0153 - val_accuracy: 0.9206
Epoch 5/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1191 - accuracy: 0.7333 - val_loss: 0.0136 - val_accuracy: 0.9242
Epoch 6/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1144 - accuracy: 0.7366 - val_loss: 0.0157 - val_accuracy: 0.9065
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1148 - accuracy: 0.7373 - val_loss: 0.0129 - val_accuracy: 0.9259
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1248 - accuracy: 0.7377 - val_loss: 0.0161 - val_accuracy: 0.9140
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1130 - accuracy: 0.7385 - val_loss: 0.0116 - val_accuracy: 0.9246
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1136 - accuracy: 0.7379 - val_loss: 0.0128 - val_accuracy: 0.9265
2938/2938 [==============================] - 2s 551us/step
Epoch 1/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.4304 - accuracy: 0.5351 - val_loss: 0.0231 - val_accuracy: 0.9057
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1340 - accuracy: 0.7215 - val_loss: 0.0176 - val_accuracy: 0.9148
Epoch 3/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1254 - accuracy: 0.7280 - val_loss: 0.0157 - val_accuracy: 0.9158
Epoch 4/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1206 - accuracy: 0.7312 - val_loss: 0.0143 - val_accuracy: 0.9264
Epoch 5/10
5964/5964 [==============================] - 12s 2ms/step - loss: 0.1168 - accuracy: 0.7337 - val_loss: 0.0124 - val_accuracy: 0.9248
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1131 - accuracy: 0.7398 - val_loss: 0.0135 - val_accuracy: 0.9214
Epoch 7/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1151 - accuracy: 0.7389 - val_loss: 0.0124 - val_accuracy: 0.9268
Epoch 8/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1148 - accuracy: 0.7407 - val_loss: 0.0138 - val_accuracy: 0.9230
Epoch 9/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1145 - accuracy: 0.7422 - val_loss: 0.0126 - val_accuracy: 0.9245
Epoch 10/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1105 - accuracy: 0.7425 - val_loss: 0.0150 - val_accuracy: 0.9243
2938/2938 [==============================] - 2s 562us/step
Epoch 1/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.4332 - accuracy: 0.5407 - val_loss: 0.0232 - val_accuracy: 0.9062
Epoch 2/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1355 - accuracy: 0.7184 - val_loss: 0.0182 - val_accuracy: 0.9128
Epoch 3/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1269 - accuracy: 0.7259 - val_loss: 0.0147 - val_accuracy: 0.9247
Epoch 4/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1200 - accuracy: 0.7315 - val_loss: 0.0150 - val_accuracy: 0.9249
Epoch 5/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1181 - accuracy: 0.7363 - val_loss: 0.0124 - val_accuracy: 0.9257
Epoch 6/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1141 - accuracy: 0.7394 - val_loss: 0.0153 - val_accuracy: 0.9120
Epoch 7/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1125 - accuracy: 0.7406 - val_loss: 0.0127 - val_accuracy: 0.9282
Epoch 8/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1279 - accuracy: 0.7382 - val_loss: 0.0151 - val_accuracy: 0.9163
Epoch 9/10
5964/5964 [==============================] - 11s 2ms/step - loss: 0.1172 - accuracy: 0.7421 - val_loss: 0.0124 - val_accuracy: 0.9215
Epoch 10/10
5964/5964 [==============================] - 10s 2ms/step - loss: 0.1096 - accuracy: 0.7419 - val_loss: 0.0160 - val_accuracy: 0.9158
2938/2938 [==============================] - 2s 515us/step
Mean average precision over 10 runs: 0.0417
Out[51]:
[0.0301,
 0.0311,
 0.0495,
 0.0254,
 0.0294,
 0.0546,
 0.0368,
 0.0234,
 0.0366,
 0.1002]

Results¶

In [52]:
print(f'Mean average precision over 10 runs:   {round(np.mean(test_scores),4)}')
print(f'Coefficient of variation over 10 runs: {round(np.std(test_scores)/np.mean(test_scores),4)}')
[round(x,4) for x in test_scores]
Mean average precision over 10 runs:   0.0417
Coefficient of variation over 10 runs: 0.52
Out[52]:
[0.0301,
 0.0311,
 0.0495,
 0.0254,
 0.0294,
 0.0546,
 0.0368,
 0.0234,
 0.0366,
 0.1002]

Conclusion¶

In [53]:
import datetime as dt
print("Completed: ", dt.datetime.now())
Completed:  2021-05-06 15:56:19.284316